A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
Fix Max Token ID for Qwen-VL-Chat (#11980)
Signed-off-by: Alex-Brooks <Alex.brooks@ibm.com>
Alex Brooks committed
Jan 13, 2025 at 01:37 UTC
5340a30d0193547a19e236757fec1f3f246642f9
1 file changed
+9
vllm/transformers_utils/tokenizer.py
+9
@@ -1,3 +1,4 @@
1
+import contextlib
2
import os
3
import warnings
4
from pathlib import Path
@@ -67,7 +68,15 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
68
tokenizer.all_special_tokens_extended)
69
tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
70
tokenizer_len = len(tokenizer)
71
+
72
max_token_id = max(tokenizer.get_vocab().values())
73
+ # Some tokenizers (e.g., QwenTokenizer) have special tokens that
74
+ # are added and included in the implementation of the vocab_size
75
+ # property, but not in get_vocab(); if there is an implementation
76
+ # of vocab size, we should take the greater value.
77
+ if hasattr(tokenizer, "vocab_size"):
78
+ with contextlib.suppress(NotImplementedError):
79
+ max_token_id = max(max_token_id, tokenizer.vocab_size)
80
81
class CachedTokenizer(tokenizer.__class__): # type: ignore
82