A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 11:15 UTC
[Bugfix] Fix missing first token in tool calls during reasoning-to-tool transition (#30671)
Signed-off-by: mondaylord <20212010046@fudan.edu.cn>
mondaylord committed
Dec 16, 2025 at 00:13 UTC
17fec3af0942da83bcebe2ca0cb4f6ae81c634d8
1 file changed
+28
-32
vllm/entrypoints/openai/serving_chat.py
+28
-32
@@ -964,21 +964,9 @@ class OpenAIServingChat(OpenAIServing):
964
assert reasoning_end_arr is not None
965
output_token_ids = as_list(output.token_ids)
966
if not reasoning_end_arr[i]:
967
- delta_message = (
968
- reasoning_parser.extract_reasoning_streaming(
969
- previous_text,
970
- current_text,
971
- delta_text,
972
- previous_token_ids,
973
- current_token_ids,
974
- output_token_ids,
975
- )
976
- )
967
# When encountering think end id in prompt_token_ids
968
# i.e {"enable_thinking": False},
969
# set reasoning status to end.
980
- # Remove the text and token ids related
981
- # to 'reasoning'.
970
if (
971
res.prompt_token_ids
972
and reasoning_parser.is_reasoning_end(
@@ -987,30 +975,38 @@ class OpenAIServingChat(OpenAIServing):
975
):
976
reasoning_end_arr[i] = True
977
current_token_ids = output_token_ids
990
- if delta_message and delta_message.content:
991
- current_text = delta_message.content
992
- delta_message.content = None
993
- else:
994
- current_text = ""
995
- # When encountering think end id in delta_token_ids,
996
- # set reasoning status to end.
997
- # Remove the text and token ids related
998
- # to 'reasoning'.
999
- if reasoning_parser.is_reasoning_end(output_token_ids):
1000
- reasoning_end_arr[i] = True
1001
- current_token_ids = (
1002
- reasoning_parser.extract_content_ids(
1003
- output_token_ids
978
+ # Don't update current_text, keep it as is from delta
979
+ else:
980
+ delta_message = (
981
+ reasoning_parser.extract_reasoning_streaming(
982
+ previous_text,
983
+ current_text,
984
+ delta_text,
985
+ previous_token_ids,
986
+ current_token_ids,
987
+ output_token_ids,
988
)
989
)
1006
- if delta_message and delta_message.content:
1007
- current_text = delta_message.content
1008
- delta_message.content = None
1009
- else:
1010
- current_text = ""
990
+
991
+ # When encountering think end id in delta_token_ids,
992
+ # set reasoning status to end.
993
+ # Remove the text and token ids related
994
+ # to 'reasoning'.
995
+ if reasoning_parser.is_reasoning_end(output_token_ids):
996
+ reasoning_end_arr[i] = True
997
+ current_token_ids = (
998
+ reasoning_parser.extract_content_ids(
999
+ output_token_ids
1000
+ )
1001
+ )
1002
+ if delta_message and delta_message.content:
1003
+ current_text = delta_message.content
1004
+ delta_message.content = None
1005
+ else:
1006
+ current_text = ""
1007
1008
# handle tool calls only after reasoning is done,
1013
- else:
1009
+ if reasoning_end_arr[i]:
1010
delta_token_ids = output_token_ids
1011
# First time to tool call,
1012
# add the remaining text and token ids