A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 22:45 UTC
[Perf] Optimize chat completion streaming performance (#33782)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Feb 4, 2026 at 20:30 UTC
f67ee8b859215df4b521c67b9f26e27f30c9739f
1 file changed
+17
-14
vllm/entrypoints/openai/chat_completion/serving.py
+17
-14
@@ -679,6 +679,7 @@ class OpenAIServingChat(OpenAIServing):
679
# For reasoning parser and tool call all enabled
680
added_content_delta_arr = [False] * num_choices
681
reasoning_end_arr = [False] * num_choices
682
+ prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices
683
else:
684
all_previous_token_ids = None
685
@@ -824,6 +825,16 @@ class OpenAIServingChat(OpenAIServing):
825
i = output.index
826
tool_parser = tool_parsers[i]
827
828
+ if (
829
+ self.reasoning_parser
830
+ and res.prompt_token_ids
831
+ and prompt_is_reasoning_end_arr[i] is None
832
+ ):
833
+ # only check once per choice, because prompt_token_ids
834
+ # are the same for all deltas in that choice
835
+ prompt_is_reasoning_end_arr[i] = (
836
+ reasoning_parser.is_reasoning_end(res.prompt_token_ids)
837
+ )
838
if finish_reason_sent[i]:
839
continue
840
@@ -926,13 +937,11 @@ class OpenAIServingChat(OpenAIServing):
937
# i.e {"enable_thinking": False},
938
# set reasoning status to end.
939
# Only keep 'content', remove 'reasoning'.
929
- if reasoning_parser.is_reasoning_end(
930
- as_list(output.token_ids)
931
- ) or (
932
- res.prompt_token_ids
933
- and reasoning_parser.is_reasoning_end(
934
- res.prompt_token_ids
940
+ if (
941
+ reasoning_parser.is_reasoning_end(
942
+ as_list(output.token_ids)
943
)
944
+ or prompt_is_reasoning_end_arr[i]
945
):
946
reasoning_end_arr[i] = True
947
if delta_message and delta_message.content:
@@ -991,8 +1000,7 @@ class OpenAIServingChat(OpenAIServing):
1000
if (
1001
self.reasoning_parser is not None
1002
and not reasoning_end_arr[i]
994
- and res.prompt_token_ids
995
- and reasoning_parser.is_reasoning_end(res.prompt_token_ids)
1003
+ and prompt_is_reasoning_end_arr[i]
1004
):
1005
reasoning_end_arr[i] = True
1006
@@ -1049,12 +1057,7 @@ class OpenAIServingChat(OpenAIServing):
1057
# When encountering think end id in prompt_token_ids
1058
# i.e {"enable_thinking": False},
1059
# set reasoning status to end.
1052
- if (
1053
- res.prompt_token_ids
1054
- and reasoning_parser.is_reasoning_end(
1055
- res.prompt_token_ids
1056
- )
1057
- ):
1060
+ if prompt_is_reasoning_end_arr[i]:
1061
reasoning_end_arr[i] = True
1062
current_token_ids = output_token_ids
1063
# Don't update current_text, keep it as is from delta