A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[BugFix] Work-around incremental detokenization edge case error (#19449)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Jun 11, 2025 at 23:43 UTC
d5bdf899e4ea7db68f731af6be0635b54de4adb3
2 files changed
+113
-6
tests/v1/engine/test_fast_incdec_prefix_err.py
new
+80
@@ -0,0 +1,80 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from transformers import AutoTokenizer
5
+
6
+from vllm.sampling_params import SamplingParams
7
+from vllm.v1.engine import EngineCoreRequest
8
+from vllm.v1.engine.detokenizer import IncrementalDetokenizer
9
+
10
+# ruff: noqa: E501
11
+
12
+
13
+def test_fast_inc_detok_invalid_utf8_err_case():
14
+ """
15
+ Test edge case where tokenizer can produce non-monotonic,
16
+ invalid UTF-8 output, which breaks the internal state of
17
+ tokenizers' DecodeStream.
18
+ See https://github.com/vllm-project/vllm/issues/17448.
19
+
20
+ Thanks to reproducer from @fpaupier:
21
+ https://gist.github.com/fpaupier/0ed1375bd7633c5be6c894b1c7ac1be3.
22
+ """
23
+ tokenizer = AutoTokenizer.from_pretrained("google/gemma-3-1b-it")
24
+
25
+ # Create a test request
26
+ prompt_token_ids = [107, 4606, 236787, 107]
27
+ params = SamplingParams(skip_special_tokens=True)
28
+ request = EngineCoreRequest(
29
+ "test",
30
+ prompt_token_ids,
31
+ None,
32
+ None,
33
+ None,
34
+ params,
35
+ None,
36
+ 0.0,
37
+ None,
38
+ cache_salt=None,
39
+ data_parallel_rank=None,
40
+ )
41
+
42
+ detokenizer = IncrementalDetokenizer.from_new_request(tokenizer, request)
43
+
44
+ assert detokenizer.__class__.__name__ == "FastIncrementalDetokenizer", \
45
+ "Should use FastIncrementalDetokenizer by default"
46
+
47
+ # Process tokens incrementally
48
+ test_tokens = [
49
+ 236840, 107, 138, 236782, 107, 140, 236775, 6265, 1083, 623, 121908,
50
+ 147418, 827, 107, 140, 236775, 6265, 236779, 2084, 1083, 623, 203292,
51
+ 827, 107, 140, 236775, 6265, 236779, 7777, 1083, 623, 121908, 147418,
52
+ 569, 537, 236789, 65880, 569, 537, 236789, 62580, 853, 115693, 210118,
53
+ 35178, 16055, 1270, 759, 215817, 4758, 1925, 1117, 827, 107, 140,
54
+ 236775, 5654, 1083, 623, 110733, 46291, 827, 107, 140, 236775, 5654,
55
+ 236779, 2084, 1083, 623, 136955, 56731, 827, 107, 140, 236775, 5654,
56
+ 236779, 7777, 1083, 623, 194776, 2947, 496, 109811, 1608, 890, 215817,
57
+ 4758, 1925, 1117, 2789, 432, 398, 602, 31118, 569, 124866, 134772, 509,
58
+ 19478, 1640, 33779, 236743, 236770, 236819, 236825, 236771, 432, 398,
59
+ 432, 237167, 827, 107, 140, 236775, 77984, 1083, 623, 2709, 236745,
60
+ 2555, 513, 236789, 602, 31118, 569
61
+ ]
62
+
63
+ output = ""
64
+ for i, token_id in enumerate(test_tokens):
65
+ detokenizer.update([token_id], False)
66
+
67
+ finished = i == len(test_tokens) - 1
68
+ output += detokenizer.get_next_output_text(finished, delta=True)
69
+
70
+
71
+# fmt: off
72
+ assert output == r'''[
73
+ {
74
+ "source": "Résultats",
75
+ "source_type": "CONCEPT",
76
+ "source_description": "Résultats de l'analyse de l'impact des opérations israéliennes sur la frontière libanaise",
77
+ "target": "Israël",
78
+ "target_type": "ORGANIZATION",
79
+ "target_description": "Pays qui a obtenu à sa frontière libanaise « un niveau de calme inédit depuis les années 1960 »",
80
+ "relationship": "Obtention d'un niveau de'''
vllm/v1/engine/detokenizer.py
+33
-6
@@ -17,6 +17,14 @@ from vllm.v1.engine import EngineCoreRequest
17
18
logger = init_logger(__name__)
19
20
+# Only tokenizers >= 0.21.1 supports DecodeStream used for
21
+# FastIncrementalDetokenizer.
22
+USE_FAST_DETOKENIZER = version.parse(
23
+ tokenizers.__version__) >= version.parse("0.21.1")
24
+
25
+# Error string from https://github.com/huggingface/tokenizers/blob/909fdde2a4ffedd9295206f705eb612be2a91b12/tokenizers/src/tokenizer/mod.rs#L1042
26
+INVALID_PREFIX_ERR_MSG = "Invalid prefix encountered"
27
+
28
29
class IncrementalDetokenizer:
30
@@ -46,10 +54,9 @@ class IncrementalDetokenizer:
54
# No tokenizer => skipping detokenization.
55
return IncrementalDetokenizer()
56
49
- if (isinstance(tokenizer, PreTrainedTokenizerFast) and version.parse(
50
- tokenizers.__version__) >= version.parse("0.21.1")):
57
+ if USE_FAST_DETOKENIZER and isinstance(tokenizer,
58
+ PreTrainedTokenizerFast):
59
# Fast tokenizer => use tokenizers library DecodeStream.
52
- # And only tokenizers >= 0.21.1 supports Fast Detokenizer.
60
return FastIncrementalDetokenizer(tokenizer, request)
61
62
# Fall back to slow python-based incremental detokenization.
@@ -157,8 +164,11 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
164
super().__init__(request)
165
166
sampling_params = request.sampling_params
167
+
168
+ self.request_id = request.request_id
169
+ self.skip_special_tokens = sampling_params.skip_special_tokens
170
self.stream = DecodeStream(
161
- skip_special_tokens=sampling_params.skip_special_tokens)
171
+ skip_special_tokens=self.skip_special_tokens)
172
173
self.tokenizer: Tokenizer = tokenizer._tokenizer
174
@@ -174,7 +184,7 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
184
185
# Prime the stream.
186
for tid in prompt_suffix:
177
- self.stream.step(self.tokenizer, tid)
187
+ self._protected_step(tid)
188
189
self.spaces_between_special_tokens = (
190
sampling_params.skip_special_tokens
@@ -199,7 +209,7 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
209
self.spaces_between_special_tokens = True
210
211
def decode_next(self, next_token_id: int) -> str:
202
- token = self.stream.step(self.tokenizer, next_token_id)
212
+ token = self._protected_step(next_token_id)
213
214
if not self.spaces_between_special_tokens:
215
special_token = self.added_token_ids.get(next_token_id)
@@ -211,6 +221,23 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
221
222
return token or ""
223
224
+ def _protected_step(self, next_token_id: int) -> Optional[str]:
225
+ try:
226
+ token = self.stream.step(self.tokenizer, next_token_id)
227
+ except Exception as e:
228
+ if str(e) != INVALID_PREFIX_ERR_MSG:
229
+ raise e
230
+ # Recover from edge case where tokenizer can produce non-monotonic,
231
+ # invalid UTF-8 output, which breaks the internal state of
232
+ # tokenizers' DecodeStream.
233
+ # See https://github.com/vllm-project/vllm/issues/17448.
234
+ logger.warning(
235
+ "Encountered invalid prefix detokenization error"
236
+ " for request %s, resetting decode stream.", self.request_id)
237
+ self.stream = DecodeStream(self.skip_special_tokens)
238
+ token = self.stream.step(self.tokenizer, next_token_id)
239
+ return token
240
+
241
242
class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer):
243