[Bugfix][Tokenizer] Fix off-by-one max_token_id from vocab_size (#59491)
Signed-off-by: Shijie Lyu <lshjhf@gmail.com>
Shijie Lyu committed
Sep 30, 2026 at 22:43 UTC
37cecc8141e2dcba9dd534897a164d4baaf27229
2 files changed
+19
-2
tests/tokenizers_/test_hf.py
+16
@@ -65,3 +65,19 @@ def test_thread_pool_tokenizer_pickle(model_id: str):
65
66
# Idempotence: wrapping an already-pooled tokenizer returns it unchanged.
67
assert maybe_make_thread_pool(pooled_tokenizer) is pooled_tokenizer # type: ignore[type-var]
68
+
69
+
70
+@pytest.mark.parametrize("model_id", ["openai-community/gpt2"])
71
+def test_cached_tokenizer_max_token_id_is_a_valid_id(model_id: str):
72
+ """max_token_id is the largest valid id, one less than vocab_size."""
73
+ reference_tokenizer = AutoTokenizer.from_pretrained(model_id)
74
+ cached_tokenizer = get_cached_tokenizer(deepcopy(reference_tokenizer))
75
+
76
+ expected = max(
77
+ max(reference_tokenizer.get_vocab().values()),
78
+ reference_tokenizer.vocab_size - 1,
79
+ )
80
+ assert cached_tokenizer.max_token_id == expected
81
+ assert cached_tokenizer.max_token_id == max(
82
+ reference_tokenizer.get_vocab().values()
83
+ )
vllm/tokenizers/hf.py
+3
-2
@@ -124,10 +124,11 @@ def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer:
124
# Some tokenizers (e.g., QwenTokenizer) have special tokens that
125
# are added and included in the implementation of the vocab_size
126
# property, but not in get_vocab(); if there is an implementation
127
- # of vocab size, we should take the greater value.
127
+ # of vocab size, we should take the greater value. vocab_size is a
128
+ # count, so the largest id it implies is vocab_size - 1.
129
if hasattr(tokenizer, "vocab_size"):
130
with contextlib.suppress(NotImplementedError):
130
- max_token_id = max(max_token_id, tokenizer.vocab_size)
131
+ max_token_id = max(max_token_id, tokenizer.vocab_size - 1)
132
133
class CachedTokenizer(tokenizer.__class__): # type: ignore
134
@property