[Bugfix][Parser] Fix U+FFFD leak at reasoning-to-content transition in engine parsers (#46159)
Signed-off-by: Ben Browning <bbrownin@redhat.com>
Ben Browning committed
Jun 19, 2026 at 18:09 UTC
859e4d436ba0fb0da8a655a80d5c4fab12adc82e
5 files changed
+191
-7
tests/parser/engine/replay_harness.py
+3
@@ -96,6 +96,9 @@ class MockTokenizer:
96
return "".join(parts)
97
98
99
+CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
100
+
101
+
102
def make_mock_tokenizer(sample: Sample) -> MockTokenizer:
103
"""Build a mock tokenizer from a sample's vocab and token data."""
104
return MockTokenizer(
tests/parser/engine/test_delegating_replay.py
+1
-2
@@ -19,6 +19,7 @@ import pytest
19
from pydantic import TypeAdapter
20
21
from tests.parser.engine.replay_harness import (
22
+ CHUNK_SIZES,
23
MockTokenizer,
24
assert_parse_output,
25
collect_output,
@@ -113,8 +114,6 @@ _PAIRINGS = _discover_pairings()
114
115
_ALL_SAMPLES = [(p.parser_cls, s) for p in _PAIRINGS for s in p.samples]
116
116
-CHUNK_SIZES = [1, 2, 3, 5, 11, 23, None]
117
-
117
118
@pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
119
@pytest.mark.parametrize(
tests/parser/engine/test_ufffd_reasoning_transition.py
new
+181
@@ -0,0 +1,181 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Regression test for U+FFFD leak at reasoning→content transition.
4
+
5
+When byte-fallback tokens span the reasoning/content boundary,
6
+decoding isolated content-side token IDs via tokenizer.decode()
7
+produces U+FFFD (Unicode replacement character). The fix flushes
8
+the reasoning parser's engine lexer instead.
9
+
10
+Reproduces the bug at various chunk sizes and validates that the
11
+fix prevents U+FFFD from leaking into streamed content.
12
+"""
13
+
14
+from __future__ import annotations
15
+
16
+import pytest
17
+
18
+from tests.parser.engine.replay_harness import (
19
+ CHUNK_SIZES,
20
+ MockTokenizer,
21
+ collect_output,
22
+ replay_streaming,
23
+)
24
+from vllm.parser.abstract_parser import DelegatingParser
25
+from vllm.parser.engine.registered_adapters import (
26
+ Glm47MoeParserReasoningAdapter,
27
+ Glm47MoeParserToolAdapter,
28
+ Qwen3ParserReasoningAdapter,
29
+ Qwen3ParserToolAdapter,
30
+)
31
+
32
+
33
+class ByteFallbackMockTokenizer(MockTokenizer):
34
+ """MockTokenizer that returns U+FFFD for specified token IDs.
35
+
36
+ Simulates byte-fallback tokenizer behavior where isolated
37
+ partial-byte tokens decode to the Unicode replacement character.
38
+ """
39
+
40
+ def __init__(
41
+ self,
42
+ vocab: dict[str, int],
43
+ tokens: list[tuple[int, str]],
44
+ ufffd_token_ids: set[int],
45
+ ) -> None:
46
+ super().__init__(vocab, tokens)
47
+ self._ufffd_token_ids = frozenset(ufffd_token_ids)
48
+
49
+ def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
50
+ parts: list[str] = []
51
+ for tid in ids:
52
+ if skip_special_tokens and tid in self._special_ids:
53
+ continue
54
+ if tid in self._ufffd_token_ids:
55
+ parts.append("�")
56
+ else:
57
+ text = self._token_decode_map.get(tid, f"?{tid}?")
58
+ parts.append(text)
59
+ return "".join(parts)
60
+
61
+
62
+# ── Model-specific DelegatingParser subclasses ───────────────────────
63
+
64
+
65
+class _Glm47Delegating(DelegatingParser):
66
+ reasoning_parser_cls = Glm47MoeParserReasoningAdapter
67
+ tool_parser_cls = Glm47MoeParserToolAdapter
68
+
69
+
70
+class _Qwen3Delegating(DelegatingParser):
71
+ reasoning_parser_cls = Qwen3ParserReasoningAdapter
72
+ tool_parser_cls = Qwen3ParserToolAdapter
73
+
74
+
75
+# ── Shared test data ─────────────────────────────────────────────────
76
+
77
+_SHARED_TOKENS: list[tuple[int, str]] = [
78
+ (100, "Let me"),
79
+ (101, " think"),
80
+ (102, " about"),
81
+ (103, " Samsung."),
82
+ (51, "</think>"),
83
+ (200, "삼성"),
84
+ (201, "전자의"),
85
+ (202, " 주가를"),
86
+ (203, " 분석합니다."),
87
+]
88
+
89
+_SHARED_UFFFD_IDS: set[int] = {200}
90
+
91
+EXPECTED_REASONING = "Let me think about Samsung."
92
+EXPECTED_CONTENT = "삼성전자의 주가를 분석합니다."
93
+
94
+_MODEL_CONFIGS = [
95
+ pytest.param(
96
+ {
97
+ "<think>": 50,
98
+ "</think>": 51,
99
+ "<tool_call>": 60,
100
+ "</tool_call>": 61,
101
+ "<arg_key>": 62,
102
+ "</arg_key>": 63,
103
+ "<arg_value>": 64,
104
+ "</arg_value>": 65,
105
+ },
106
+ _Glm47Delegating,
107
+ id="glm47",
108
+ ),
109
+ pytest.param(
110
+ {
111
+ "<think>": 50,
112
+ "</think>": 51,
113
+ "<tool_call>": 60,
114
+ "</tool_call>": 61,
115
+ },
116
+ _Qwen3Delegating,
117
+ id="qwen3",
118
+ ),
119
+]
120
+
121
+
122
+# ── Tests ────────────────────────────────────────────────────────────
123
+
124
+
125
+class TestUfffdReasoningTransition:
126
+ """U+FFFD must not appear at the reasoning→content transition."""
127
+
128
+ @pytest.mark.parametrize("vocab,delegating_cls", _MODEL_CONFIGS)
129
+ @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
130
+ def test_no_ufffd(self, chunk_size, vocab, delegating_cls):
131
+ tokenizer = ByteFallbackMockTokenizer(vocab, _SHARED_TOKENS, _SHARED_UFFFD_IDS)
132
+ parser = delegating_cls(tokenizer)
133
+ deltas = replay_streaming(
134
+ parser,
135
+ _SHARED_TOKENS,
136
+ chunk_size=chunk_size,
137
+ finished_on_last=True,
138
+ )
139
+ output = collect_output(deltas)
140
+
141
+ assert "�" not in output.content, (
142
+ f"U+FFFD leaked into content: {output.content!r}"
143
+ )
144
+ assert output.content == EXPECTED_CONTENT
145
+ assert output.reasoning == EXPECTED_REASONING
146
+
147
+ def test_byte_fallback_tokenizer_produces_ufffd(self):
148
+ """Validate the fixture: decode() returns U+FFFD for isolated
149
+ byte-fallback token IDs, proving the old code path would leak."""
150
+ vocab = dict(_MODEL_CONFIGS[0].values[0])
151
+ tokenizer = ByteFallbackMockTokenizer(vocab, _SHARED_TOKENS, _SHARED_UFFFD_IDS)
152
+ assert tokenizer.decode([200]) == "�"
153
+
154
+ @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
155
+ def test_multiple_ufffd_tokens_at_boundary(self, chunk_size):
156
+ """Multiple consecutive byte-fallback tokens at the boundary."""
157
+ tokens: list[tuple[int, str]] = [
158
+ (100, "Reasoning."),
159
+ (51, "</think>"),
160
+ (200, "삼"),
161
+ (201, "성"),
162
+ (202, "전자"),
163
+ ]
164
+ ufffd_ids: set[int] = {200, 201}
165
+ vocab = dict(_MODEL_CONFIGS[0].values[0])
166
+
167
+ tokenizer = ByteFallbackMockTokenizer(vocab, tokens, ufffd_ids)
168
+ parser = _Glm47Delegating(tokenizer)
169
+ deltas = replay_streaming(
170
+ parser,
171
+ tokens,
172
+ chunk_size=chunk_size,
173
+ finished_on_last=True,
174
+ )
175
+ output = collect_output(deltas)
176
+
177
+ assert "�" not in output.content, (
178
+ f"U+FFFD leaked into content: {output.content!r}"
179
+ )
180
+ assert output.content == "삼성전자"
181
+ assert output.reasoning == "Reasoning."
vllm/parser/abstract_parser.py
+3
-4
@@ -794,11 +794,10 @@ class DelegatingParser(Parser):
794
reasoning_transitioned = True
795
current_token_ids = self.extract_content_ids(delta_token_ids)
796
if self._engine_based:
797
+ flush_delta = reasoning_parser.finish_streaming() # type: ignore[union-attr, attr-defined]
798
current_text = (
798
- self.model_tokenizer.decode(current_token_ids)
799
- if current_token_ids
800
- else ""
801
- )
799
+ (delta_message.content if delta_message else None) or ""
800
+ ) + ((flush_delta.content if flush_delta else None) or "")
801
if delta_message and self._tool_parser is not None:
802
delta_message.content = None
803
else:
vllm/parser/engine/parser_engine.py
+3
-1
@@ -172,7 +172,9 @@ class ParserEngine(Parser):
172
173
def finish_streaming(self) -> DeltaMessage | None:
174
events = self._engine.finish()
175
- return self._events_to_delta(events) if events else None
175
+ if events or self._deferred_content:
176
+ return self._events_to_delta(events, finished=True)
177
+ return None
178
179
def _reset(self, initial_state: ParserState | None = None) -> None:
180
self._engine.reset(initial_state=initial_state)