[Bugfix][Parser] Prevent Inkling block-end leakage with tools (#51391)
Signed-off-by: cherry77-cloud <1615405@qq.com> Co-authored-by: Ben Browning <56071+bbrowning@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
cherry77-cloud committed
Aug 9, 2026 at 00:34 UTC
75231eff2f3873e2bce7cc9558bb5227ea70b808
3 files changed
+352
-30
tests/parser/engine/test_inkling.py
+212
@@ -19,6 +19,10 @@ from tests.parser.engine.streaming_helpers import (
19
collect_function_name,
20
collect_tool_arguments,
21
)
22
+from vllm.entrypoints.openai.chat_completion.protocol import (
23
+ ChatCompletionToolsParam,
24
+ FunctionDefinition,
25
+)
26
from vllm.parser.engine.parser_engine_config import ParserState
27
from vllm.parser.inkling import InklingParser, _inkling_arg_converter
28
from vllm.parser.parser_manager import ParserManager
@@ -140,6 +144,51 @@ def _collect_reasoning(results) -> str:
144
return "".join(d.reasoning for d, _ in results if d and d.reasoning)
145
146
147
+def _function_tool(name: str = "get_weather") -> ChatCompletionToolsParam:
148
+ """A real function-tool definition, as a request would carry it."""
149
+ return ChatCompletionToolsParam(
150
+ function=FunctionDefinition(
151
+ name=name,
152
+ parameters={"type": "object", "properties": {}},
153
+ ),
154
+ )
155
+
156
+
157
+def _delegating(mock_tokenizer, tools=None):
158
+ """Build the served reasoning+tool DelegatingParser for Inkling."""
159
+ parser_cls = ParserManager.get_parser(
160
+ tool_parser_name="inkling",
161
+ reasoning_parser_name="inkling",
162
+ enable_auto_tools=True,
163
+ )
164
+ return parser_cls(mock_tokenizer, tools or [])
165
+
166
+
167
+def _stream_delegating(parser, request, text, chunk_size, prompt_token_ids):
168
+ """Stream ``text`` through ``DelegatingParser.parse_delta``, ``chunk_size``
169
+ tokens per delta; return ``(content, reasoning, ordered tool names)``."""
170
+ tokens = _tokenize(text)
171
+ content, reasoning, tools = "", "", {}
172
+ for start in range(0, len(tokens), chunk_size):
173
+ batch = tokens[start : start + chunk_size]
174
+ delta = parser.parse_delta(
175
+ "".join(t for _, t in batch),
176
+ [tid for tid, _ in batch],
177
+ request,
178
+ prompt_token_ids=prompt_token_ids if start == 0 else None,
179
+ finished=(start + chunk_size >= len(tokens)),
180
+ )
181
+ if delta and delta.content:
182
+ content += delta.content
183
+ if delta and delta.reasoning:
184
+ reasoning += delta.reasoning
185
+ if delta and delta.tool_calls:
186
+ for tc in delta.tool_calls:
187
+ if tc.function and tc.function.name:
188
+ tools[tc.index] = tc.function.name
189
+ return content, reasoning, [tools[k] for k in sorted(tools)]
190
+
191
+
192
class TestArgConverter:
193
def test_complete_wrapper(self):
194
raw = '{"name":"get_weather","args":{"city":"SF"}}'
@@ -533,3 +582,166 @@ class TestRegisteredAdapters:
582
assert result.tools_called
583
assert result.tool_calls[0].function.name == "f"
584
assert json.loads(result.tool_calls[0].function.arguments) == {"a": 1}
585
+
586
+
587
+class TestDelegatingTwoPass:
588
+ """Served reasoning+tool ``DelegatingParser`` path (issue #51387).
589
+
590
+ Unlike ``TestStreaming`` above (single engine, ``skip_tool_parsing=False``),
591
+ these drive the real two-pass parser, where the reasoning pass runs with
592
+ ``skip_tool_parsing=True`` — the path where the trailing-marker leak lived.
593
+
594
+ The bug is specific to the *tools-enabled* plain-text/reasoning path, so the
595
+ content-only cases (which emit no tool call and would still pass in a
596
+ no-tools mode) offer a real function tool on both the request and the
597
+ parser, keeping ``tool_choice="auto"``.
598
+ """
599
+
600
+ GEN_PROMPT = [_TML_VOCAB[MSG_MODEL]]
601
+
602
+ def test_plain_text_non_streaming(self, mock_tokenizer, mock_request):
603
+ tools = [_function_tool()]
604
+ mock_request.tools = tools
605
+ _, content, calls = _delegating(mock_tokenizer, tools).parse(
606
+ f"{TEXT_START}The answer is 42.{END_MESSAGE}",
607
+ mock_request,
608
+ enable_auto_tools=True,
609
+ )
610
+ assert content == "The answer is 42."
611
+ assert END_MESSAGE not in content
612
+ assert not calls
613
+
614
+ @pytest.mark.parametrize("chunk_size", [1, 3, 7, 64])
615
+ def test_plain_text_streaming(self, mock_tokenizer, mock_request, chunk_size):
616
+ tools = [_function_tool()]
617
+ mock_request.tools = tools
618
+ content, _, calls = _stream_delegating(
619
+ _delegating(mock_tokenizer, tools),
620
+ mock_request,
621
+ f"{TEXT_START}The answer is 42.{END_MESSAGE}",
622
+ chunk_size,
623
+ self.GEN_PROMPT,
624
+ )
625
+ assert content == "The answer is 42."
626
+ assert END_MESSAGE not in content
627
+ assert not calls
628
+
629
+ def test_reasoning_then_text(self, mock_tokenizer, mock_request):
630
+ tools = [_function_tool()]
631
+ mock_request.tools = tools
632
+ reasoning, content, _ = _delegating(mock_tokenizer, tools).parse(
633
+ f"{THINK_START}plan{END_MESSAGE}"
634
+ f"{MSG_MODEL}{TEXT_START}Let me look.{END_MESSAGE}",
635
+ mock_request,
636
+ enable_auto_tools=True,
637
+ )
638
+ assert reasoning == "plan"
639
+ assert content == "Let me look."
640
+ assert END_MESSAGE not in content
641
+
642
+ def test_multi_tool_round_trip(self, mock_tokenizer, mock_request):
643
+ _, _, tools = _delegating(mock_tokenizer).parse(
644
+ _tool_block("f", '{"a":1}') + MSG_MODEL + _tool_block("g", '{"b":2}'),
645
+ mock_request,
646
+ enable_auto_tools=True,
647
+ )
648
+ assert [t.name for t in tools] == ["f", "g"]
649
+
650
+ def test_text_then_tool(self, mock_tokenizer, mock_request):
651
+ _, content, tools = _delegating(mock_tokenizer).parse(
652
+ f"{TEXT_START}intro{END_MESSAGE}{MSG_MODEL}" + _tool_block("f", "{}"),
653
+ mock_request,
654
+ enable_auto_tools=True,
655
+ )
656
+ assert content == "intro"
657
+ assert [t.name for t in tools] == ["f"]
658
+
659
+ def test_tool_then_text(self, mock_tokenizer, mock_request):
660
+ _, content, tools = _delegating(mock_tokenizer).parse(
661
+ _tool_block("f", "{}") + MSG_MODEL + f"{TEXT_START}done{END_MESSAGE}",
662
+ mock_request,
663
+ enable_auto_tools=True,
664
+ )
665
+ assert content == "done"
666
+ assert [t.name for t in tools] == ["f"]
667
+
668
+ def test_two_tools_then_text(self, mock_tokenizer, mock_request):
669
+ _, content, tools = _delegating(mock_tokenizer).parse(
670
+ _tool_block("f", "{}")
671
+ + MSG_MODEL
672
+ + _tool_block("g", "{}")
673
+ + MSG_MODEL
674
+ + f"{TEXT_START}after{END_MESSAGE}",
675
+ mock_request,
676
+ enable_auto_tools=True,
677
+ )
678
+ assert content == "after"
679
+ assert [t.name for t in tools] == ["f", "g"]
680
+
681
+ def test_reasoning_then_tool_non_streaming(self, mock_tokenizer, mock_request):
682
+ reasoning, _, tools = _delegating(mock_tokenizer).parse(
683
+ f"{THINK_START}think{END_MESSAGE}{MSG_MODEL}" + _tool_block("f", '{"x":1}'),
684
+ mock_request,
685
+ enable_auto_tools=True,
686
+ )
687
+ assert reasoning == "think"
688
+ assert [t.name for t in tools] == ["f"]
689
+
690
+ def test_end_sampling_text_closer_consumed(self, mock_tokenizer, mock_request):
691
+ tools = [_function_tool()]
692
+ mock_request.tools = tools
693
+ _, content, calls = _delegating(mock_tokenizer, tools).parse(
694
+ f"{TEXT_START}hi{END_SAMPLING}",
695
+ mock_request,
696
+ enable_auto_tools=True,
697
+ )
698
+ assert content == "hi"
699
+ assert END_SAMPLING not in content
700
+ assert not calls
701
+
702
+ def test_end_sampling_tool_closer_round_trips(self, mock_tokenizer, mock_request):
703
+ _, _, tools = _delegating(mock_tokenizer).parse(
704
+ f'{TOOL_JSON}{{"name":"f","args":{{}}}}{END_SAMPLING}',
705
+ mock_request,
706
+ enable_auto_tools=True,
707
+ )
708
+ assert [t.name for t in tools] == ["f"]
709
+
710
+ def test_incomplete_tool_at_eos(self, mock_tokenizer, mock_request):
711
+ _, _, tools = _delegating(mock_tokenizer).parse(
712
+ f'{TOOL_JSON}{{"name":"d","args":{{"k":"v"',
713
+ mock_request,
714
+ enable_auto_tools=True,
715
+ )
716
+ assert [t.name for t in tools] == ["d"]
717
+
718
+ def test_reset_reuse_after_incomplete_span(self, mock_tokenizer, mock_request):
719
+ parser = _delegating(mock_tokenizer)
720
+ parser.parse(
721
+ f'{TOOL_JSON}{{"name":"d","args":{{"k":"v"',
722
+ mock_request,
723
+ enable_auto_tools=True,
724
+ )
725
+ _, content, _ = parser.parse(
726
+ f"{TEXT_START}fresh{END_MESSAGE}",
727
+ mock_request,
728
+ enable_auto_tools=True,
729
+ )
730
+ assert content == "fresh"
731
+ assert END_MESSAGE not in content
732
+
733
+ @pytest.mark.parametrize("chunk_size", [1, 3, 64])
734
+ def test_reasoning_then_tool_streaming(
735
+ self, mock_tokenizer, mock_request, chunk_size
736
+ ):
737
+ content, _, tools = _stream_delegating(
738
+ _delegating(mock_tokenizer),
739
+ mock_request,
740
+ f"{THINK_START}plan{END_MESSAGE}{MSG_MODEL}"
741
+ + _tool_block("get_weather", '{"city":"SF"}'),
742
+ chunk_size,
743
+ self.GEN_PROMPT,
744
+ )
745
+ assert tools == ["get_weather"]
746
+ assert TOOL_JSON not in content
747
+ assert END_MESSAGE not in content
tests/parser/engine/test_parser_engine.py
+85
@@ -1075,6 +1075,91 @@ class TestReasoningOnlyEndTokenLeak:
1075
assert "Hi!" in d2.content
1076
1077
1078
+class TestSkipToolSpanForwarding:
1079
+ """Reasoning (skip_tool_parsing) pass: tool syntax is forwarded verbatim
1080
+ for the tool pass, and the lexical passthrough flag never goes stale.
1081
+
1082
+ ``_combined_config`` defines ``</tool_call>`` only from ``TOOL_ARGS``; the
1083
+ skip pass stays in ``CONTENT``, so the closer arrives with no current-state
1084
+ transition. It must still be forwarded and must clear the span — otherwise
1085
+ ``_in_skipped_tool_span`` stays True for the rest of the request. This is the
1086
+ shape of every current engine grammar whose wrapper closer has no CONTENT
1087
+ transition (qwen3, deepseek, glm47_moe, nemotron_v3).
1088
+ """
1089
+
1090
+ _TOOL_VOCAB = {**_VOCAB, '{"a":1}': 300}
1091
+ _TOOL_CALL = '<tool_call>{"a":1}</tool_call>'
1092
+ _TOOL_IDS = [202, 300, 203]
1093
+
1094
+ def _skip_engine(self):
1095
+ engine = _make_engine(
1096
+ vocab=self._TOOL_VOCAB,
1097
+ special_tokens=list(_VOCAB.keys()),
1098
+ )
1099
+ engine._engine.skip_tool_parsing = True
1100
+ engine._engine.reset(initial_state=ParserState.CONTENT)
1101
+ return engine
1102
+
1103
+ def test_tool_syntax_forwarded_and_span_cleared(self):
1104
+ engine = self._skip_engine()
1105
+ events = engine._engine.feed(self._TOOL_CALL, self._TOOL_IDS)
1106
+ forwarded = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
1107
+ assert forwarded == self._TOOL_CALL
1108
+ assert engine._engine._in_skipped_tool_span is False
1109
+
1110
+ def test_span_not_stale_across_two_tool_calls(self):
1111
+ engine = self._skip_engine()
1112
+ engine._engine.feed(self._TOOL_CALL, self._TOOL_IDS)
1113
+ assert engine._engine._in_skipped_tool_span is False
1114
+ events = engine._engine.feed(self._TOOL_CALL, self._TOOL_IDS)
1115
+ forwarded = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
1116
+ assert forwarded == self._TOOL_CALL
1117
+ assert engine._engine._in_skipped_tool_span is False
1118
+
1119
+ def test_content_after_transitionless_closer_observable(self):
1120
+ """Observable lifecycle contract (not just the private flag): after a
1121
+ tool closer with no CONTENT transition, a later *shared* block-ender
1122
+ that also closes text blocks must be consumed, not leaked as content.
1123
+
1124
+ The config combines the two real patterns — a qwen3-style tool-only
1125
+ closer with no CONTENT transition (``</a>``) and an inkling-style token
1126
+ that closes both tool and text blocks (``</b>``). A stale span would
1127
+ route the trailing ``</b>`` down the forward path and leak it.
1128
+ """
1129
+ ts, close_a, close_b = "<tc>", "</a>", "</b>"
1130
+ cfg = ParserEngineConfig(
1131
+ name="shared_closer_test",
1132
+ terminals={"TOOL_START": ts, "CLOSE_A": close_a, "CLOSE_B": close_b},
1133
+ transitions={
1134
+ (ParserState.CONTENT, "TOOL_START"): Transition(
1135
+ ParserState.TOOL_ARGS, (EventType.TOOL_CALL_START,)
1136
+ ),
1137
+ (ParserState.TOOL_ARGS, "CLOSE_A"): Transition(
1138
+ ParserState.CONTENT, (EventType.TOOL_CALL_END,)
1139
+ ),
1140
+ (ParserState.TOOL_ARGS, "CLOSE_B"): Transition(
1141
+ ParserState.CONTENT, (EventType.TOOL_CALL_END,)
1142
+ ),
1143
+ (ParserState.CONTENT, "CLOSE_B"): Transition(ParserState.CONTENT, ()),
1144
+ },
1145
+ initial_state=ParserState.CONTENT,
1146
+ content_events={
1147
+ ParserState.CONTENT: EventType.TEXT_CHUNK,
1148
+ ParserState.TOOL_ARGS: EventType.ARG_VALUE_CHUNK,
1149
+ },
1150
+ )
1151
+ engine = _make_engine(config=cfg, vocab={ts: 210, close_a: 211, close_b: 212})
1152
+ engine._engine.skip_tool_parsing = True
1153
+ engine._engine.reset(initial_state=ParserState.CONTENT)
1154
+
1155
+ events = engine._engine.feed(f"{ts}args{close_a}text{close_b}", [])
1156
+ content = "".join(e.value for e in events if e.type == EventType.TEXT_CHUNK)
1157
+
1158
+ assert content == f"{ts}args{close_a}text"
1159
+ assert close_b not in content
1160
+ assert engine._engine._in_skipped_tool_span is False
1161
+
1162
+
1163
# ── TestToolAdapterForwardsKwargs ──────────────────────────────────
1164
1165
vllm/parser/engine/streaming_parser_engine.py
+55
-30
@@ -156,6 +156,13 @@ class StreamingParserEngine:
156
for (state, terminal), tr in config.transitions.items()
157
if tr.next_state in self._TOOL_STATES or state in self._TOOL_STATES
158
)
159
+ # TOOL_CALL_END may close an inner call rather than its lexical wrapper,
160
+ # as in MiniMax, so identify exits from state transitions instead.
161
+ self._tool_exit_terminals: frozenset[str] = frozenset(
162
+ terminal
163
+ for (state, terminal), tr in config.transitions.items()
164
+ if state in self._TOOL_STATES and tr.next_state not in self._TOOL_STATES
165
+ )
166
167
self.skip_tool_parsing = False
168
self.reset(initial_state=initial_state)
@@ -186,6 +193,7 @@ class StreamingParserEngine:
193
self._scanner.reset()
194
self._lexer.reset()
195
self._message_header_buffer = ""
196
+ self._in_skipped_tool_span = False
197
self._reset_args_state()
198
199
def feed(
@@ -316,39 +324,56 @@ class StreamingParserEngine:
324
if transition is None:
325
if self._has_drops and terminal == DROP_TERMINAL:
326
return []
327
+ # The projected skip state may not define the wrapper closer.
328
+ if self.skip_tool_parsing and terminal in self._tool_exit_terminals:
329
+ self._in_skipped_tool_span = False
330
return self._emit_for_state(value)
331
332
if self.skip_tool_parsing and terminal in self._tool_terminals:
322
- if self.state == ParserState.MESSAGE_HEADER:
323
- self.state = ParserState.CONTENT
324
- self._message_header_buffer = ""
325
- return [
326
- SemanticEvent(
327
- EventType.TEXT_CHUNK,
328
- value=value,
329
- tool_index=self.tool_index,
330
- )
331
- ]
332
- if EventType.REASONING_END in transition.events:
333
- self.state = ParserState.CONTENT
334
- return [
335
- SemanticEvent(
336
- EventType.REASONING_END,
337
- value=value,
338
- tool_index=self.tool_index,
339
- ),
340
- SemanticEvent(
341
- EventType.TEXT_CHUNK,
342
- value=value,
343
- tool_index=self.tool_index,
344
- ),
345
- ]
346
- content_type = self.config.content_events.get(self.state)
347
- if content_type is not None:
348
- return [
349
- SemanticEvent(content_type, value=value, tool_index=self.tool_index)
350
- ]
351
- return []
333
+ # Inkling reuses one terminal for tool, text, and reasoning exits.
334
+ # Outside a forwarded tool span, apply its normal transition.
335
+ is_opener = transition.next_state in self._TOOL_STATES
336
+ is_exit = terminal in self._tool_exit_terminals
337
+ used_as_plain_closer = (
338
+ is_exit and not is_opener and not self._in_skipped_tool_span
339
+ )
340
+ if not used_as_plain_closer:
341
+ if is_opener:
342
+ self._in_skipped_tool_span = True
343
+ elif is_exit:
344
+ self._in_skipped_tool_span = False
345
+ if self.state == ParserState.MESSAGE_HEADER:
346
+ self.state = ParserState.CONTENT
347
+ self._message_header_buffer = ""
348
+ return [
349
+ SemanticEvent(
350
+ EventType.TEXT_CHUNK,
351
+ value=value,
352
+ tool_index=self.tool_index,
353
+ )
354
+ ]
355
+ if EventType.REASONING_END in transition.events:
356
+ self.state = ParserState.CONTENT
357
+ return [
358
+ SemanticEvent(
359
+ EventType.REASONING_END,
360
+ value=value,
361
+ tool_index=self.tool_index,
362
+ ),
363
+ SemanticEvent(
364
+ EventType.TEXT_CHUNK,
365
+ value=value,
366
+ tool_index=self.tool_index,
367
+ ),
368
+ ]
369
+ content_type = self.config.content_events.get(self.state)
370
+ if content_type is not None:
371
+ return [
372
+ SemanticEvent(
373
+ content_type, value=value, tool_index=self.tool_index
374
+ )
375
+ ]
376
+ return []
377
378
if transition.skip_in_token_id_mode and self._ever_had_token_ids:
379
return self._emit_for_state(value)