@tduncan / Nvidia-Optimization-Stack-VLLM / commits / e7cf4730d6

[Bugfix] Drop incomplete tool-call markup in non-streaming to match streaming (#47562)

Signed-off-by: Suvrakamal Das <jyb2084.usa@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Ben Browning <56071+bbrowning@users.noreply.github.com>

Suvrakamal Das committed Sep 2, 2026 at 03:11 UTC e7cf4730d6c441723faec060b99cf597b701f285

6 files changed +399 -1

tests/parser/engine/test_inkling.py

+110

@@ -869,3 +869,113 @@ def test_content_tool_start_emits_reasoning_end_in_reasoning_pass():

869 EventType.TEXT_CHUNK,

870 ]

871 assert events[1].value == TOOL_JSON

872 +

873 +

874 +def test_skip_reasoning_parsing_inert_for_shared_markers():

875 + """``skip_reasoning_parsing`` may only bypass reasoning-exclusive

876 + markers. Inkling has none — ``<|end_message|>`` is labelled THINK_END

877 + yet also closes text, header, and tool blocks — so the flag must be

878 + inert: a tool block still closes through its transition and following

879 + text returns to CONTENT instead of leaking into the argument stream."""

880 + engine = StreamingParserEngine(inkling_config(), tokenizer=None)

881 + engine.skip_reasoning_parsing = True

882 + engine.reset(initial_state=ParserState.CONTENT)

883 + events = engine.parse_complete(

884 + f'{TOOL_JSON}{{"name":"f","args":{{}}}}{END_MESSAGE}after'

885 + )

886 + types = [e.type for e in events]

887 + end_idx = types.index(EventType.TOOL_CALL_END)

888 + text_after = [e.value for e in events[end_idx:] if e.type == EventType.TEXT_CHUNK]

889 + assert text_after == ["after"]

890 +

891 +

892 +class TestToolParserWithoutReasoningParser:

893 + """Inkling served with only the tool parser, no ``--reasoning-parser``.

894 +

895 + That configuration turns on ``skip_reasoning_parsing``, and Inkling

896 + labels ``<|end_message|>`` as THINK_END; a bypass keyed on the label

897 + would neutralize the closer of every block kind, leaking markers into

898 + content and leaving tool calls unterminated. Structure must keep

899 + parsing exactly as with the reasoning parser attached."""

900 +

901 + GEN_PROMPT = [_TML_VOCAB[MSG_MODEL]]

902 +

903 + def _tool_only(self, mock_tokenizer, tools=None):

904 + parser_cls = ParserManager.get_parser(

905 + tool_parser_name="inkling",

906 + enable_auto_tools=True,

907 + )

908 + return parser_cls(mock_tokenizer, tools or [])

909 +

910 + def test_plain_text_non_streaming(self, mock_tokenizer, mock_request):

911 + tools = [_function_tool()]

912 + mock_request.tools = tools

913 + _, content, calls = self._tool_only(mock_tokenizer, tools).parse(

914 + f"{TEXT_START}The answer is 42.{END_MESSAGE}",

915 + mock_request,

916 + enable_auto_tools=True,

917 + )

918 + assert content == "The answer is 42."

919 + assert not calls

920 +

921 + @pytest.mark.parametrize("chunk_size", [1, 3, 64])

922 + def test_plain_text_streaming(self, mock_tokenizer, mock_request, chunk_size):

923 + tools = [_function_tool()]

924 + mock_request.tools = tools

925 + content, _, names, _ = _stream_delegating(

926 + self._tool_only(mock_tokenizer, tools),

927 + mock_request,

928 + f"{TEXT_START}The answer is 42.{END_MESSAGE}",

929 + chunk_size,

930 + self.GEN_PROMPT,

931 + )

932 + assert content == "The answer is 42."

933 + assert not names

934 +

935 + def test_tool_block_non_streaming(self, mock_tokenizer, mock_request):

936 + _, content, calls = self._tool_only(mock_tokenizer).parse(

937 + _tool_block("get_weather", '{"city":"SF"}'),

938 + mock_request,

939 + enable_auto_tools=True,

940 + )

941 + assert [c.name for c in calls] == ["get_weather"]

942 + assert not content

943 +

944 + @pytest.mark.parametrize("chunk_size", [1, 3, 64])

945 + def test_tool_block_streaming(self, mock_tokenizer, mock_request, chunk_size):

946 + tools = [_function_tool()]

947 + mock_request.tools = tools

948 + content, _, names, args = _stream_delegating(

949 + self._tool_only(mock_tokenizer, tools),

950 + mock_request,

951 + _tool_block("get_weather", '{"city":"SF"}'),

952 + chunk_size,

953 + self.GEN_PROMPT,

954 + )

955 + assert names == ["get_weather"]

956 + assert json.loads(args[0]) == {"city": "SF"}

957 + assert content == ""

958 +

959 + def test_thinking_then_text_keeps_structure(self, mock_tokenizer, mock_request):

960 + tools = [_function_tool()]

961 + mock_request.tools = tools

962 + _, content, calls = self._tool_only(mock_tokenizer, tools).parse(

963 + f"{THINK_START}plan{END_MESSAGE}"

964 + f"{MSG_MODEL}{TEXT_START}Let me look.{END_MESSAGE}",

965 + mock_request,

966 + enable_auto_tools=True,

967 + )

968 + assert content == "Let me look."

969 + assert END_MESSAGE not in content

970 + assert THINK_START not in content

971 + assert not calls

972 +

973 + def test_thinking_then_tool_still_promotes(self, mock_tokenizer, mock_request):

974 + _, content, calls = self._tool_only(mock_tokenizer).parse(

975 + f"{THINK_START}think{END_MESSAGE}{MSG_MODEL}"

976 + + _tool_block("get_weather", '{"x":1}'),

977 + mock_request,

978 + enable_auto_tools=True,

979 + )

980 + assert [c.name for c in calls] == ["get_weather"]

981 + assert not content

tests/parser/engine/test_parser_engine.py

+197

@@ -1909,3 +1909,200 @@ class TestDropSpecialTokens:

1909 e.value for e in events if e.type == EventType.REASONING_CHUNK

1910 )

1911 assert "<bos>" not in reasoning_text

1912 +

1913 +

1914 +# ── TestTruncatedToolOpenerStreamParity ──────────────────────────────

1915 +

1916 +

1917 +class TestTruncatedToolOpenerStreamParity:

1918 + """Regression tests for #47137: when generation terminates inside a

1919 + ``<tool_call>`` opener that has not been promoted to a tool call

1920 + (via ``max_tokens`` or a ``stop`` string), the non-streaming path

1921 + must drop the incomplete markup, matching the streaming path."""

1922 +

1923 + _QWEN3_VOCAB = {

1924 + "<tool_call>": 100,

1925 + "</tool_call>": 101,

1926 + }

1927 +

1928 + def _make_parser(self):

1929 + from vllm.parser.parser_manager import ParserManager

1930 +

1931 + parser_cls = ParserManager.get_parser(

1932 + tool_parser_name="qwen3_coder",

1933 + enable_auto_tools=True,

1934 + )

1935 + tokenizer = make_mock_tokenizer(self._QWEN3_VOCAB)

1936 + return parser_cls(tokenizer, [])

1937 +

1938 + def _stream_content(self, request, chunks: list[str]) -> str:

1939 + parser = self._make_parser()

1940 + content = ""

1941 + for i, chunk in enumerate(chunks):

1942 + delta_token_ids = [

1943 + tid for text, tid in self._QWEN3_VOCAB.items() if text in chunk

1944 + ]

1945 + delta = parser.parse_delta(

1946 + chunk,

1947 + delta_token_ids,

1948 + request,

1949 + prompt_token_ids=[1],

1950 + finished=(i == len(chunks) - 1),

1951 + )

1952 + if delta and delta.content:

1953 + content += delta.content

1954 + return content

1955 +

1956 + @pytest.mark.parametrize(

1957 + "chunks",

1958 + [

1959 + ["<tool_call>"],

1960 + ["<tool_call>", "\n"],

1961 + ["<tool_call>", "\n", "<"],

1962 + ["<tool_call>", "\n", "<function"],

1963 + ],

1964 + )

1965 + def test_truncated_opener_dropped_in_both_paths(self, mock_request, chunks):

1966 + """A cutoff inside the opener yields no content and no tool calls

1967 + in both the non-streaming and streaming paths."""

1968 + text = "".join(chunks)

1969 +

1970 + _, content, tool_calls = self._make_parser().parse(

1971 + text, mock_request, enable_auto_tools=True

1972 + )

1973 + streamed = self._stream_content(mock_request, chunks)

1974 +

1975 + assert not tool_calls

1976 + assert (content or "") == streamed == ""

1977 +

1978 + def test_content_before_truncated_opener_preserved(self, mock_request):

1979 + """Only the incomplete markup is dropped; content generated before

1980 + the opener is returned identically by both paths."""

1981 + chunks = ["Checking the weather. ", "<tool_call>", "\n", "<function"]

1982 + text = "".join(chunks)

1983 +

1984 + _, content, tool_calls = self._make_parser().parse(

1985 + text, mock_request, enable_auto_tools=True

1986 + )

1987 + streamed = self._stream_content(mock_request, chunks)

1988 +

1989 + assert not tool_calls

1990 + assert content == streamed == "Checking the weather. "

1991 +

1992 + def test_content_around_unpromoted_tool_block_stays_ordered(self, mock_request):

1993 + """Text surrounding a complete-but-unparsable tool block keeps its

1994 + original order in the non-streaming path, matching streaming."""

1995 + chunks = ["A ", "<tool_call>", "garbage", "</tool_call>", " B"]

1996 + text = "".join(chunks)

1997 +

1998 + _, content, tool_calls = self._make_parser().parse(

1999 + text, mock_request, enable_auto_tools=True

2000 + )

2001 + streamed = self._stream_content(mock_request, chunks)

2002 +

2003 + assert not tool_calls

2004 + assert content == streamed == "A B"

2005 +

2006 + def test_complete_tool_call_still_promoted(self, mock_request):

2007 + """Sanity check: a complete tool call still parses in the

2008 + non-streaming path after the truncation fix."""

2009 + text = (

2010 + "<tool_call>\n"

2011 + "<function=get_weather>\n"

2012 + "<parameter=city>Tokyo</parameter>\n"

2013 + "</function>\n"

2014 + "</tool_call>"

2015 + )

2016 + _, content, tool_calls = self._make_parser().parse(

2017 + text, mock_request, enable_auto_tools=True

2018 + )

2019 + assert tool_calls is not None and len(tool_calls) == 1

2020 + assert tool_calls[0].name == "get_weather"

2021 + assert not content

2022 +

2023 +

2024 +class TestThinkMarkupWithoutReasoningParser:

2025 + """With no reasoning parser configured, reasoning markup is plain

2026 + content: engine-backed tool parsers must return it verbatim in both

2027 + the non-streaming and streaming paths, not consume the markers or

2028 + reclassify the block as reasoning and drop it."""

2029 +

2030 + _VOCAB = {

2031 + "<think>": 90,

2032 + "</think>": 91,

2033 + "<tool_call>": 100,

2034 + "</tool_call>": 101,

2035 + }

2036 +

2037 + # Two distinct failure modes are covered: qwen3_coder consumed a

2038 + # stray </think> marker in CONTENT state (unbalanced markup),

2039 + # deepseek_v4 re-entered REASONING on <think> and dropped the block

2040 + # from non-streaming content entirely.

2041 + _PARSERS = ["qwen3_coder", "deepseek_v4"]

2042 +

2043 + def _make_parser(self, tool_parser_name):

2044 + from vllm.parser.parser_manager import ParserManager

2045 +

2046 + parser_cls = ParserManager.get_parser(

2047 + tool_parser_name=tool_parser_name,

2048 + enable_auto_tools=True,

2049 + )

2050 + tokenizer = make_mock_tokenizer(self._VOCAB)

2051 + return parser_cls(tokenizer, [])

2052 +

2053 + def _stream(self, tool_parser_name, request, chunks):

2054 + parser = self._make_parser(tool_parser_name)

2055 + content, reasoning = "", ""

2056 + for i, chunk in enumerate(chunks):

2057 + delta_token_ids = [

2058 + tid for text, tid in self._VOCAB.items() if text in chunk

2059 + ]

2060 + delta = parser.parse_delta(

2061 + chunk,

2062 + delta_token_ids,

2063 + request,

2064 + prompt_token_ids=[1],

2065 + finished=(i == len(chunks) - 1),

2066 + )

2067 + if delta and delta.content:

2068 + content += delta.content

2069 + if delta and delta.reasoning:

2070 + reasoning += delta.reasoning

2071 + return content, reasoning

2072 +

2073 + @pytest.mark.parametrize("tool_parser_name", _PARSERS)

2074 + def test_think_block_passes_through_both_paths(

2075 + self, mock_request, tool_parser_name

2076 + ):

2077 + chunks = ["<think>", "reasoning here", "</think>", "the answer"]

2078 + text = "".join(chunks)

2079 +

2080 + reasoning, content, tool_calls = self._make_parser(tool_parser_name).parse(

2081 + text, mock_request, enable_auto_tools=True

2082 + )

2083 + streamed_content, streamed_reasoning = self._stream(

2084 + tool_parser_name, mock_request, chunks

2085 + )

2086 +

2087 + assert not tool_calls

2088 + assert reasoning is None

2089 + assert streamed_reasoning == ""

2090 + assert content == streamed_content == text

2091 +

2092 + def test_think_block_preserved_alongside_tool_call(self, mock_request):

2093 + """The tools-called branch also returns the tool parser's content;

2094 + a think block preceding a promoted tool call must survive in it."""

2095 + think = "<think>plan</think>"

2096 + text = think + (

2097 + "<tool_call>\n"

2098 + "<function=get_weather>\n"

2099 + "<parameter=city>Tokyo</parameter>\n"

2100 + "</function>\n"

2101 + "</tool_call>"

2102 + )

2103 + _, content, tool_calls = self._make_parser("qwen3_coder").parse(

2104 + text, mock_request, enable_auto_tools=True

2105 + )

2106 + assert tool_calls is not None and len(tool_calls) == 1

2107 + assert tool_calls[0].name == "get_weather"

2108 + assert content == think

vllm/parser/abstract_parser.py

+19

@@ -131,6 +131,17 @@ class Parser:

131 self._reasoning_parser is None

132 or self._reasoning_parser.engine_based_streaming

133 ) and (self._tool_parser is None or self._tool_parser.engine_based_streaming)

134 + if (

135 + self._reasoning_parser is None

136 + and self._tool_parser is not None

137 + and hasattr(self._tool_parser, "skip_reasoning_parsing")

138 + ):

139 + # With no reasoning parser configured, reasoning markup is

140 + # plain content: an engine-based tool parser should pass it

141 + # through verbatim where its grammar allows, not consume or

142 + # reclassify it. The engine ignores the flag for markers

143 + # shared with non-reasoning structure.

144 + self._tool_parser.skip_reasoning_parsing = True

145 self._stream_state = StreamState(

146 tool_call_id_type=(

147 get_tool_call_id_type(model_config)

@@ -508,6 +519,14 @@ class DelegatingParser(Parser):

519 or (isinstance(content, str) and not content.strip())

520 ):

521 return [], None

522 + # No complete tool calls: for engine-based parsers, return

523 + # the tool parser's content, which drops incomplete

524 + # tool-call markup (e.g. a <tool_call> opener truncated by

525 + # max_tokens or a stop string), so the non-streaming path

526 + # matches streaming. Legacy parsers keep their existing

527 + # behavior of returning the raw content.

528 + if self._engine_based and tool_call_info is not None:

529 + return None, tool_call_info.content or None

530 return None, content

531

532 return tool_calls, content

vllm/parser/engine/adapters.py

+17

@@ -150,6 +150,15 @@ class ParserEngineToolAdapter(ToolParser):

150 state so it can parse reasoning-stripped content (i.e. the output of

151 :meth:`ReasoningParser.extract_reasoning`).

152

153 + When no reasoning parser is configured at the serving layer, the

154 + owning :class:`~vllm.parser.abstract_parser.Parser` sets

155 + :attr:`skip_reasoning_parsing` so reasoning markup — plain content

156 + in that configuration — passes through verbatim instead of being

157 + consumed or reclassified by the engine. The engine honors the flag

158 + only for markers that are reasoning-exclusive in its config; a

159 + grammar like Inkling, whose block closer is shared with text and

160 + tool blocks, keeps parsing unchanged.

161 +

162 Subclasses set :attr:`_parser_engine_cls` to the concrete

163 :class:`ParserEngine` class.

164 """

@@ -166,6 +175,14 @@ class ParserEngineToolAdapter(ToolParser):

175 super().__init__(tokenizer, tools)

176 self._parser_engine = self._parser_engine_cls(tokenizer, tools, **kwargs) # type: ignore[call-arg]

177

178 + @property

179 + def skip_reasoning_parsing(self) -> bool:

180 + return self._parser_engine.skip_reasoning_parsing

181 +

182 + @skip_reasoning_parsing.setter

183 + def skip_reasoning_parsing(self, value: bool) -> None:

184 + self._parser_engine.skip_reasoning_parsing = value

185 +

186 def adjust_request(

187 self,

188 request: ChatCompletionRequest | ResponsesRequest,

vllm/parser/engine/parser_engine.py

+17 -1

@@ -175,6 +175,14 @@ class ParserEngine(Parser):

175 def skip_tool_parsing(self, value: bool) -> None:

176 self._engine.skip_tool_parsing = value

177

178 + @property

179 + def skip_reasoning_parsing(self) -> bool:

180 + return self._engine.skip_reasoning_parsing

181 +

182 + @skip_reasoning_parsing.setter

183 + def skip_reasoning_parsing(self, value: bool) -> None:

184 + self._engine.skip_reasoning_parsing = value

185 +

186 @property

187 def reasoning_ended(self) -> bool:

188 return self._reasoning_ended

@@ -714,6 +722,7 @@ class ParserEngine(Parser):

722 content_parts: list[str] = []

723 reasoning_parts: list[str] = []

724

725 + carried_deferred = self._deferred_content

726 seen_tool_event = False

727 suppress = self._suppress_tool_calls

728 for event in events:

@@ -750,7 +759,14 @@ class ParserEngine(Parser):

759 tool_call_deltas = self._coalesce_tool_call_deltas(tool_call_deltas)

760

761 if self._deferred_content and (not seen_tool_event or not tool_call_deltas):

753 - content_parts.insert(0, self._deferred_content)

762 + # Deferred content carried in from a previous delta precedes this

763 + # delta's content; content deferred during this call (text after

764 + # an unpromoted tool event) follows it.

765 + deferred_now = self._deferred_content[len(carried_deferred) :]

766 + if carried_deferred:

767 + content_parts.insert(0, carried_deferred)

768 + if deferred_now:

769 + content_parts.append(deferred_now)

770 self._deferred_content = ""

771

772 content_str = "".join(content_parts)

vllm/parser/engine/streaming_parser_engine.py

+39

@@ -164,7 +164,12 @@ class StreamingParserEngine:

164 if state in self._TOOL_STATES and tr.next_state not in self._TOOL_STATES

165 )

166

167 + self._reasoning_markup_terminals: frozenset[str] = (

168 + self._compute_reasoning_markup_terminals()

169 + )

170 +

171 self.skip_tool_parsing = False

172 + self.skip_reasoning_parsing = False

173 self.reset(initial_state=initial_state)

174

175 @property

@@ -358,6 +363,37 @@ class StreamingParserEngine:

363 }

364 )

365

366 + _PLAIN_STATES = frozenset({ParserState.CONTENT, ParserState.REASONING})

367 +

368 + _REASONING_EVENTS = frozenset({EventType.REASONING_START, EventType.REASONING_END})

369 +

370 + def _compute_reasoning_markup_terminals(self) -> frozenset[str]:

371 + """Terminals the ``skip_reasoning_parsing`` bypass may neutralize.

372 +

373 + Only reasoning-exclusive markers qualify: every transition they

374 + participate in stays within CONTENT/REASONING and emits nothing

375 + but reasoning events. Inkling's ``<|end_message|>`` is labelled

376 + THINK_END yet also closes text, header, and tool blocks;

377 + bypassing a shared marker would eat that structure, so one impure

378 + marker disables the bypass for the whole config.

379 + """

380 + markers = frozenset(

381 + terminal

382 + for (state, terminal), tr in self.config.transitions.items()

383 + if ParserState.REASONING in (state, tr.next_state)

384 + and tr.next_state not in self._TOOL_STATES

385 + )

386 + for (state, terminal), tr in self.config.transitions.items():

387 + if terminal not in markers:

388 + continue

389 + if (

390 + state not in self._PLAIN_STATES

391 + or tr.next_state not in self._PLAIN_STATES

392 + or not self._REASONING_EVENTS.issuperset(tr.events)

393 + ):

394 + return frozenset()

395 + return markers

396 +

397 def _on_terminal(

398 self, terminal: str, value: str, token_count: int = 0

399 ) -> list[SemanticEvent]:

@@ -372,6 +408,9 @@ class StreamingParserEngine:

408 self._in_skipped_tool_span = False

409 return self._emit_for_state(value, token_count)

410

411 + if self.skip_reasoning_parsing and terminal in self._reasoning_markup_terminals:

412 + return self._emit_for_state(value, token_count)

413 +

414 if self.skip_tool_parsing and terminal in self._tool_terminals:

415 # Inkling reuses one terminal for tool, text, and reasoning exits.

416 # Outside a forwarded tool span, apply its normal transition.