@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5b0ed8391d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] using len(tokenizer) instead of tokenizer.vocab_size in AllowedTokenIdsLogitsProcessor (#11156)

zhangjf committed Dec 13, 2024 at 23:56 UTC 5b0ed8391d497439595a1968d65df93da98265ca
1 file changed +2 -2
vllm/entrypoints/openai/logits_processors.py
+2 -2
@@ -71,7 +71,7 @@ def get_logits_processors(
71
72 # Check if token_id is within the vocab size
73 for token_id, bias in clamped_logit_bias.items():
74 - if token_id < 0 or token_id >= tokenizer.vocab_size:
74 + if token_id < 0 or token_id >= len(tokenizer):
75 raise ValueError(f"token_id {token_id} in logit_bias contains "
76 "out-of-vocab token id")
77
@@ -81,6 +81,6 @@ def get_logits_processors(
81 if allowed_token_ids is not None:
82 logits_processors.append(
83 _get_allowed_token_ids_logits_processor(
84 - frozenset(allowed_token_ids), tokenizer.vocab_size))
84 + frozenset(allowed_token_ids), len(tokenizer)))
85
86 return logits_processors