A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix Incremental Detokenization with `tokenizers == 0.22.0` (#24159)
Signed-off-by: Fanli Lin <fanli.lin@intel.com> Signed-off-by: Fanli Lin <fanli0116@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Fanli Lin committed
Sep 4, 2025 at 17:47 UTC
2c301ee2eb2d60015936c5e34c80fa62d3c2d37d
1 file changed
+3
-2
vllm/v1/engine/detokenizer.py
+3
-2
@@ -234,7 +234,7 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
234
try:
235
token = self.stream.step(self.tokenizer, next_token_id)
236
except Exception as e:
237
- if str(e) != INVALID_PREFIX_ERR_MSG:
237
+ if not str(e).startswith(INVALID_PREFIX_ERR_MSG):
238
raise e
239
# Recover from edge case where tokenizer can produce non-monotonic,
240
# invalid UTF-8 output, which breaks the internal state of
@@ -243,7 +243,8 @@ class FastIncrementalDetokenizer(BaseIncrementalDetokenizer):
243
logger.warning(
244
"Encountered invalid prefix detokenization error"
245
" for request %s, resetting decode stream.", self.request_id)
246
- self.stream = DecodeStream(self.skip_special_tokens)
246
+ self.stream = DecodeStream(
247
+ skip_special_tokens=self.skip_special_tokens)
248
token = self.stream.step(self.tokenizer, next_token_id)
249
return token
250