A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
Only turn on FastIncrementalDetokenizer when tokenizers >= 0.21.1 (#17158)
Lu Fang committed
Apr 25, 2025 at 02:10 UTC
fc966e9cc6a6e01f642c721514de008a06a57203
1 file changed
+5
-1
vllm/v1/engine/detokenizer.py
+5
-1
@@ -2,6 +2,8 @@
2
from abc import ABC, abstractmethod
3
from typing import Optional
4
5
+import tokenizers
6
+from packaging import version
7
from tokenizers import Tokenizer
8
from tokenizers.decoders import DecodeStream
9
from transformers import PreTrainedTokenizerFast
@@ -43,8 +45,10 @@ class IncrementalDetokenizer:
45
# No tokenizer => skipping detokenization.
46
return IncrementalDetokenizer()
47
46
- if isinstance(tokenizer, PreTrainedTokenizerFast):
48
+ if (isinstance(tokenizer, PreTrainedTokenizerFast) and version.parse(
49
+ tokenizers.__version__) >= version.parse("0.21.1")):
50
# Fast tokenizer => use tokenizers library DecodeStream.
51
+ # And only tokenizers >= 0.21.1 supports Fast Detokenizer.
52
return FastIncrementalDetokenizer(tokenizer, request)
53
54
# Fall back to slow python-based incremental detokenization.