@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 203d4f82ac
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Core][Bugfix] cache len of tokenizer (#3741)

youkaichao committed Mar 29, 2024 at 18:46 UTC 203d4f82ac137f04c0e487bbdbba86b3cf3ae7bf
1 file changed +4
vllm/transformers_utils/tokenizer.py
+4
@@ -26,6 +26,7 @@ def get_cached_tokenizer(
26 tokenizer_all_special_tokens_extended = (
27 tokenizer.all_special_tokens_extended)
28 tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
29 + tokenizer_len = len(tokenizer)
30
31 class CachedTokenizer(tokenizer.__class__):
32
@@ -41,6 +42,9 @@ def get_cached_tokenizer(
42 def all_special_tokens_extended(self):
43 return tokenizer_all_special_tokens_extended
44
45 + def __len__(self):
46 + return tokenizer_len
47 +
48 CachedTokenizer.__name__ = f"Cached{tokenizer.__class__.__name__}"
49
50 tokenizer.__class__ = CachedTokenizer