A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Core][Bugfix] cache len of tokenizer (#3741)
youkaichao committed
Mar 29, 2024 at 18:46 UTC
203d4f82ac137f04c0e487bbdbba86b3cf3ae7bf
1 file changed
+4
vllm/transformers_utils/tokenizer.py
+4
@@ -26,6 +26,7 @@ def get_cached_tokenizer(
26
tokenizer_all_special_tokens_extended = (
27
tokenizer.all_special_tokens_extended)
28
tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
29
+ tokenizer_len = len(tokenizer)
30
31
class CachedTokenizer(tokenizer.__class__):
32
@@ -41,6 +42,9 @@ def get_cached_tokenizer(
42
def all_special_tokens_extended(self):
43
return tokenizer_all_special_tokens_extended
44
45
+ def __len__(self):
46
+ return tokenizer_len
47
+
48
CachedTokenizer.__name__ = f"Cached{tokenizer.__class__.__name__}"
49
50
tokenizer.__class__ = CachedTokenizer