A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Bugfix] fix KeyError on top logprobs are special tokens (#17637)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
May 5, 2025 at 10:22 UTC
5394ad738720ab9c2dfb0d00fcd894ac27a4fd67
1 file changed
+2
-1
vllm/entrypoints/openai/serving_chat.py
+2
-1
@@ -1111,7 +1111,8 @@ class OpenAIServingChat(OpenAIServing):
1111
return_as_token_id is not None else self.return_tokens_as_token_ids
1112
for i, token_id in enumerate(token_ids):
1113
step_top_logprobs = top_logprobs[i]
1114
- if step_top_logprobs is None:
1114
+ if step_top_logprobs is None or step_top_logprobs.get(
1115
+ token_id) is None:
1116
token = tokenizer.decode(token_id)
1117
if should_return_as_token_id:
1118
token = f"token_id:{token_id}"