@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5340a30d01
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

Fix Max Token ID for Qwen-VL-Chat (#11980)

Signed-off-by: Alex-Brooks <Alex.brooks@ibm.com>

Alex Brooks committed Jan 13, 2025 at 01:37 UTC 5340a30d0193547a19e236757fec1f3f246642f9
1 file changed +9
vllm/transformers_utils/tokenizer.py
+9
@@ -1,3 +1,4 @@
1 +import contextlib
2 import os
3 import warnings
4 from pathlib import Path
@@ -67,7 +68,15 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer:
68 tokenizer.all_special_tokens_extended)
69 tokenizer_all_special_tokens = set(tokenizer.all_special_tokens)
70 tokenizer_len = len(tokenizer)
71 +
72 max_token_id = max(tokenizer.get_vocab().values())
73 + # Some tokenizers (e.g., QwenTokenizer) have special tokens that
74 + # are added and included in the implementation of the vocab_size
75 + # property, but not in get_vocab(); if there is an implementation
76 + # of vocab size, we should take the greater value.
77 + if hasattr(tokenizer, "vocab_size"):
78 + with contextlib.suppress(NotImplementedError):
79 + max_token_id = max(max_token_id, tokenizer.vocab_size)
80
81 class CachedTokenizer(tokenizer.__class__): # type: ignore
82