A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] using len(tokenizer) instead of tokenizer.vocab_size in AllowedTokenIdsLogitsProcessor (#11156)
zhangjf committed
Dec 13, 2024 at 23:56 UTC
5b0ed8391d497439595a1968d65df93da98265ca
1 file changed
+2
-2
vllm/entrypoints/openai/logits_processors.py
+2
-2
@@ -71,7 +71,7 @@ def get_logits_processors(
71
72
# Check if token_id is within the vocab size
73
for token_id, bias in clamped_logit_bias.items():
74
- if token_id < 0 or token_id >= tokenizer.vocab_size:
74
+ if token_id < 0 or token_id >= len(tokenizer):
75
raise ValueError(f"token_id {token_id} in logit_bias contains "
76
"out-of-vocab token id")
77
@@ -81,6 +81,6 @@ def get_logits_processors(
81
if allowed_token_ids is not None:
82
logits_processors.append(
83
_get_allowed_token_ids_logits_processor(
84
- frozenset(allowed_token_ids), tokenizer.vocab_size))
84
+ frozenset(allowed_token_ids), len(tokenizer)))
85
86
return logits_processors