@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 195 lines 3.97 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 from transformers import AutoTokenizer
5
6 from vllm.sampling_params import SamplingParams
7 from vllm.v1.engine import EngineCoreRequest
8 from vllm.v1.engine.detokenizer import IncrementalDetokenizer
9
10 # ruff: noqa: E501
11
12
13 def test_fast_inc_detok_invalid_utf8_err_case():
14 """Test edge case where tokenizer can produce non-monotonic,
15 invalid UTF-8 output, which breaks the internal state of
16 tokenizers' DecodeStream.
17 See https://github.com/vllm-project/vllm/issues/17448.
18
19 Thanks to reproducer from @fpaupier:
20 https://gist.github.com/fpaupier/0ed1375bd7633c5be6c894b1c7ac1be3.
21 """
22 tokenizer = AutoTokenizer.from_pretrained("google/gemma-3-1b-it")
23
24 # Create a test request
25 prompt_token_ids = [107, 4606, 236787, 107]
26 params = SamplingParams(skip_special_tokens=True)
27 request = EngineCoreRequest(
28 request_id="test",
29 external_req_id="test-ext",
30 prompt_token_ids=prompt_token_ids,
31 mm_features=None,
32 sampling_params=params,
33 pooling_params=None,
34 arrival_time=0.0,
35 lora_request=None,
36 cache_salt=None,
37 data_parallel_rank=None,
38 )
39
40 detokenizer = IncrementalDetokenizer.from_new_request(tokenizer, request)
41
42 assert detokenizer.__class__.__name__ == "FastIncrementalDetokenizer", (
43 "Should use FastIncrementalDetokenizer by default"
44 )
45
46 # Process tokens incrementally
47 test_tokens = [
48 236840,
49 107,
50 138,
51 236782,
52 107,
53 140,
54 236775,
55 6265,
56 1083,
57 623,
58 121908,
59 147418,
60 827,
61 107,
62 140,
63 236775,
64 6265,
65 236779,
66 2084,
67 1083,
68 623,
69 203292,
70 827,
71 107,
72 140,
73 236775,
74 6265,
75 236779,
76 7777,
77 1083,
78 623,
79 121908,
80 147418,
81 569,
82 537,
83 236789,
84 65880,
85 569,
86 537,
87 236789,
88 62580,
89 853,
90 115693,
91 210118,
92 35178,
93 16055,
94 1270,
95 759,
96 215817,
97 4758,
98 1925,
99 1117,
100 827,
101 107,
102 140,
103 236775,
104 5654,
105 1083,
106 623,
107 110733,
108 46291,
109 827,
110 107,
111 140,
112 236775,
113 5654,
114 236779,
115 2084,
116 1083,
117 623,
118 136955,
119 56731,
120 827,
121 107,
122 140,
123 236775,
124 5654,
125 236779,
126 7777,
127 1083,
128 623,
129 194776,
130 2947,
131 496,
132 109811,
133 1608,
134 890,
135 215817,
136 4758,
137 1925,
138 1117,
139 2789,
140 432,
141 398,
142 602,
143 31118,
144 569,
145 124866,
146 134772,
147 509,
148 19478,
149 1640,
150 33779,
151 236743,
152 236770,
153 236819,
154 236825,
155 236771,
156 432,
157 398,
158 432,
159 237167,
160 827,
161 107,
162 140,
163 236775,
164 77984,
165 1083,
166 623,
167 2709,
168 236745,
169 2555,
170 513,
171 236789,
172 602,
173 31118,
174 569,
175 ]
176
177 output = ""
178 for i, token_id in enumerate(test_tokens):
179 detokenizer.update([token_id], False)
180
181 finished = i == len(test_tokens) - 1
182 output += detokenizer.get_next_output_text(finished, delta=True)
183
184 assert (
185 output
186 == r"""[
187 {
188 "source": "Résultats",
189 "source_type": "CONCEPT",
190 "source_description": "Résultats de l'analyse de l'impact des opérations israéliennes sur la frontière libanaise",
191 "target": "Israël",
192 "target_type": "ORGANIZATION",
193 "target_description": "Pays qui a obtenu à sa frontière libanaise « un niveau de calme inédit depuis les années 1960 »",
194 "relationship": "Obtention d'un niveau de"""
195 )