[Bugfix][Parser] Fix special tokens (EOS/BOS) leaking into reasoning content (#48748)
Signed-off-by: Ben Browning <56071+bbrowning@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Ben Browning committed
Jul 22, 2026 at 16:10 UTC
3de4b2bf3c477513afff4a58680eb00d557bb53a
3 files changed
+72
-15
tests/parser/engine/test_deepseek_v4.py
+42
@@ -920,3 +920,45 @@ class TestDelegatingParserLargeDelta:
920
assert output.tool_calls[0]["name"] == "get_weather"
921
args = json.loads(output.tool_calls[0]["arguments"])
922
assert args == {"location": "Berlin"}
923
+
924
+ @pytest.mark.parametrize(
925
+ "chunk_size",
926
+ [1, 2, 3, 5, None],
927
+ ids=lambda c: f"chunk={c}",
928
+ )
929
+ def test_eos_not_leaked_when_reasoning_never_ends(self, chunk_size):
930
+ """EOS must not leak into reasoning_content when the model never
931
+ emits </think> (generation ends while still in REASONING state)."""
932
+ eos_text = "<｜end▁of▁sentence｜>"
933
+ eos_id = 128801
934
+ vocab = {
935
+ **_DSV4_FULL_VOCAB,
936
+ eos_text: eos_id,
937
+ }
938
+
939
+ reasoning_text = "Good morning! How can I help you today?"
940
+ tokens: list[tuple[int, str]] = []
941
+ tid = 100
942
+ for word in reasoning_text.split(" "):
943
+ prefix = " " if tokens else ""
944
+ tokens.append((tid, prefix + word))
945
+ tid += 1
946
+ tokens.append((eos_id, eos_text))
947
+
948
+ tokenizer = MockTokenizer(vocab=vocab, tokens=tokens)
949
+ parser = _DeepSeekV4Delegating(
950
+ tokenizer,
951
+ chat_template_kwargs={"thinking": True},
952
+ )
953
+ deltas = replay_streaming(
954
+ parser,
955
+ tokens,
956
+ chunk_size=chunk_size,
957
+ finished_on_last=True,
958
+ )
959
+ output = collect_output(deltas)
960
+
961
+ assert reasoning_text in output.reasoning
962
+ assert eos_text not in output.reasoning
963
+ assert output.content == ""
964
+ assert output.tool_calls == []
tests/parser/engine/test_parser_engine.py
+29
-7
@@ -1624,19 +1624,41 @@ class TestDropSpecialTokens:
1624
assert delta is not None
1625
assert "<bos>" in delta.reasoning
1626
1627
- def test_drops_suppressed_with_skip_tool_parsing(self):
1628
- """When skip_tool_parsing is active, drop tokens are preserved
1629
- as content so a later tool-call pass can see them."""
1627
+ def test_drops_applied_with_skip_tool_parsing(self):
1628
+ """Drop tokens are always dropped, even with skip_tool_parsing.
1629
+ DROP_TERMINALs have no transitions by construction, so no parser
1630
+ pass can use them."""
1631
+ for initial_state in (ParserState.REASONING, ParserState.CONTENT):
1632
+ engine = _make_engine(
1633
+ vocab=_DROP_VOCAB,
1634
+ special_tokens=list(_DROP_VOCAB.keys()),
1635
+ )
1636
+ engine._engine.skip_tool_parsing = True
1637
+ engine._engine.reset(initial_state=initial_state)
1638
+ events = engine._engine.feed("hello<bos>world", [72, 204, 73])
1639
+ delta = engine._events_to_delta(events)
1640
+ assert delta is not None
1641
+ output = (delta.reasoning or "") + (delta.content or "")
1642
+ assert "<bos>" not in output, f"<bos> leaked in state {initial_state}"
1643
+
1644
+ def test_transitions_unaffected_by_drop_in_reasoning_with_skip_tool_parsing(self):
1645
+ """With skip_tool_parsing in REASONING state, drop tokens are
1646
+ removed but configured terminals still fire their transitions."""
1647
engine = _make_engine(
1648
vocab=_DROP_VOCAB,
1649
special_tokens=list(_DROP_VOCAB.keys()),
1650
)
1651
engine._engine.skip_tool_parsing = True
1652
engine._engine.reset()
1636
- events = engine._engine.feed("hello<bos>world", [72, 204, 73])
1637
- delta = engine._events_to_delta(events)
1638
- assert delta is not None
1639
- assert "<bos>" in delta.reasoning
1653
+ events = engine._engine.feed("thought<bos></think>answer", [72, 204, 201, 73])
1654
+ types = [e.type for e in events]
1655
+ assert EventType.REASONING_CHUNK in types
1656
+ assert EventType.REASONING_END in types
1657
+ assert EventType.TEXT_CHUNK in types
1658
+ reasoning_text = "".join(
1659
+ e.value for e in events if e.type == EventType.REASONING_CHUNK
1660
+ )
1661
+ assert "<bos>" not in reasoning_text
1662
1663
def test_drops_in_tool_args_state(self):
1664
"""Drop tokens in TOOL_ARGS state are silently discarded."""
vllm/parser/engine/streaming_parser_engine.py
+1
-8
@@ -304,14 +304,7 @@ class StreamingParserEngine:
304
transition = self.config.transitions.get(key)
305
306
if transition is None:
307
- if (
308
- self._has_drops
309
- and terminal == DROP_TERMINAL
310
- # Preserve drop tokens when skip_tool_parsing is active so
311
- # the reasoning pass doesn't silently remove tokens that a
312
- # later tool-call pass might need to see.
313
- and not self.skip_tool_parsing
314
- ):
307
+ if self._has_drops and terminal == DROP_TERMINAL:
308
return []
309
return self._emit_for_state(value)
310