A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 01:00 UTC
[Bugfix] Missing tokens in `return_token_ids` when tool parsers is enabled in streaming mode (#29074)
Signed-off-by: Peng-YM <1048217874pengym@gmail.com>
Peng-YM committed
Dec 5, 2025 at 03:09 UTC
48a5fff66e78985a634abac0d8d7f271da744000
1 file changed
+8
-3
vllm/entrypoints/openai/serving_chat.py
+8
-3
@@ -1072,10 +1072,15 @@ class OpenAIServingChat(OpenAIServing):
1072
# wasn't ready to send a token, then
1073
# get the next token without streaming a chunk
1074
if delta_message is None:
1075
- if output.finish_reason is None:
1075
+ # NOTE: If return_token_ids is enabled, we still need to
1076
+ # send a chunk with token_ids even if delta_message is None
1077
+ # to ensure all tokens are included in the response
1078
+ if (
1079
+ output.finish_reason is None
1080
+ and not request.return_token_ids
1081
+ ):
1082
continue
1077
- else:
1078
- delta_message = DeltaMessage()
1083
+ delta_message = DeltaMessage()
1084
1085
# Log streaming delta if output logging is enabled
1086
if self.enable_log_outputs and self.request_logger: