@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 181 lines 5.87 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """Regression test for U+FFFD leak at reasoning→content transition.
4
5 When byte-fallback tokens span the reasoning/content boundary,
6 decoding isolated content-side token IDs via tokenizer.decode()
7 produces U+FFFD (Unicode replacement character). The fix flushes
8 the reasoning parser's engine lexer instead.
9
10 Reproduces the bug at various chunk sizes and validates that the
11 fix prevents U+FFFD from leaking into streamed content.
12 """
13
14 from __future__ import annotations
15
16 import pytest
17
18 from tests.parser.engine.replay_harness import (
19 CHUNK_SIZES,
20 MockTokenizer,
21 collect_output,
22 replay_streaming,
23 )
24 from vllm.parser.abstract_parser import DelegatingParser
25 from vllm.parser.engine.registered_adapters import (
26 Glm47MoeParserReasoningAdapter,
27 Glm47MoeParserToolAdapter,
28 Qwen3ParserReasoningAdapter,
29 Qwen3ParserToolAdapter,
30 )
31
32
33 class ByteFallbackMockTokenizer(MockTokenizer):
34 """MockTokenizer that returns U+FFFD for specified token IDs.
35
36 Simulates byte-fallback tokenizer behavior where isolated
37 partial-byte tokens decode to the Unicode replacement character.
38 """
39
40 def __init__(
41 self,
42 vocab: dict[str, int],
43 tokens: list[tuple[int, str]],
44 ufffd_token_ids: set[int],
45 ) -> None:
46 super().__init__(vocab, tokens)
47 self._ufffd_token_ids = frozenset(ufffd_token_ids)
48
49 def decode(self, ids: list[int], skip_special_tokens: bool = False) -> str:
50 parts: list[str] = []
51 for tid in ids:
52 if skip_special_tokens and tid in self._special_ids:
53 continue
54 if tid in self._ufffd_token_ids:
55 parts.append("�")
56 else:
57 text = self._token_decode_map.get(tid, f"?{tid}?")
58 parts.append(text)
59 return "".join(parts)
60
61
62 # ── Model-specific DelegatingParser subclasses ───────────────────────
63
64
65 class _Glm47Delegating(DelegatingParser):
66 reasoning_parser_cls = Glm47MoeParserReasoningAdapter
67 tool_parser_cls = Glm47MoeParserToolAdapter
68
69
70 class _Qwen3Delegating(DelegatingParser):
71 reasoning_parser_cls = Qwen3ParserReasoningAdapter
72 tool_parser_cls = Qwen3ParserToolAdapter
73
74
75 # ── Shared test data ─────────────────────────────────────────────────
76
77 _SHARED_TOKENS: list[tuple[int, str]] = [
78 (100, "Let me"),
79 (101, " think"),
80 (102, " about"),
81 (103, " Samsung."),
82 (51, "</think>"),
83 (200, "삼성"),
84 (201, "전자의"),
85 (202, " 주가를"),
86 (203, " 분석합니다."),
87 ]
88
89 _SHARED_UFFFD_IDS: set[int] = {200}
90
91 EXPECTED_REASONING = "Let me think about Samsung."
92 EXPECTED_CONTENT = "삼성전자의 주가를 분석합니다."
93
94 _MODEL_CONFIGS = [
95 pytest.param(
96 {
97 "<think>": 50,
98 "</think>": 51,
99 "<tool_call>": 60,
100 "</tool_call>": 61,
101 "<arg_key>": 62,
102 "</arg_key>": 63,
103 "<arg_value>": 64,
104 "</arg_value>": 65,
105 },
106 _Glm47Delegating,
107 id="glm47",
108 ),
109 pytest.param(
110 {
111 "<think>": 50,
112 "</think>": 51,
113 "<tool_call>": 60,
114 "</tool_call>": 61,
115 },
116 _Qwen3Delegating,
117 id="qwen3",
118 ),
119 ]
120
121
122 # ── Tests ────────────────────────────────────────────────────────────
123
124
125 class TestUfffdReasoningTransition:
126 """U+FFFD must not appear at the reasoning→content transition."""
127
128 @pytest.mark.parametrize("vocab,delegating_cls", _MODEL_CONFIGS)
129 @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
130 def test_no_ufffd(self, chunk_size, vocab, delegating_cls):
131 tokenizer = ByteFallbackMockTokenizer(vocab, _SHARED_TOKENS, _SHARED_UFFFD_IDS)
132 parser = delegating_cls(tokenizer)
133 deltas = replay_streaming(
134 parser,
135 _SHARED_TOKENS,
136 chunk_size=chunk_size,
137 finished_on_last=True,
138 )
139 output = collect_output(deltas)
140
141 assert "�" not in output.content, (
142 f"U+FFFD leaked into content: {output.content!r}"
143 )
144 assert output.content == EXPECTED_CONTENT
145 assert output.reasoning == EXPECTED_REASONING
146
147 def test_byte_fallback_tokenizer_produces_ufffd(self):
148 """Validate the fixture: decode() returns U+FFFD for isolated
149 byte-fallback token IDs, proving the old code path would leak."""
150 vocab = dict(_MODEL_CONFIGS[0].values[0])
151 tokenizer = ByteFallbackMockTokenizer(vocab, _SHARED_TOKENS, _SHARED_UFFFD_IDS)
152 assert tokenizer.decode([200]) == "�"
153
154 @pytest.mark.parametrize("chunk_size", CHUNK_SIZES, ids=lambda c: f"chunk={c}")
155 def test_multiple_ufffd_tokens_at_boundary(self, chunk_size):
156 """Multiple consecutive byte-fallback tokens at the boundary."""
157 tokens: list[tuple[int, str]] = [
158 (100, "Reasoning."),
159 (51, "</think>"),
160 (200, "삼"),
161 (201, "성"),
162 (202, "전자"),
163 ]
164 ufffd_ids: set[int] = {200, 201}
165 vocab = dict(_MODEL_CONFIGS[0].values[0])
166
167 tokenizer = ByteFallbackMockTokenizer(vocab, tokens, ufffd_ids)
168 parser = _Glm47Delegating(tokenizer)
169 deltas = replay_streaming(
170 parser,
171 tokens,
172 chunk_size=chunk_size,
173 finished_on_last=True,
174 )
175 output = collect_output(deltas)
176
177 assert "�" not in output.content, (
178 f"U+FFFD leaked into content: {output.content!r}"
179 )
180 assert output.content == "삼성전자"
181 assert output.reasoning == "Reasoning."