A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[BugFix] Fix tool call finish reason in streaming case (#9209)
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Maximilien de Bayser committed
Oct 11, 2024 at 22:24 UTC
ec10cb8511b7e30b8ff86caab2e4272ff3ceddca
1 file changed
+15
-11
vllm/entrypoints/openai/serving_chat.py
+15
-11
@@ -538,10 +538,12 @@ class OpenAIServingChat(OpenAIServing):
538
# any tokens that were generated but previously
539
# matched by partial json parsing
540
# only happens if we are NOT using guided decoding
541
+ auto_tools_called = False
542
if tool_parser:
542
- index = len(
543
- tool_parser.prev_tool_call_arr) - 1 if len(
544
- tool_parser.prev_tool_call_arr) > 0 else 0
543
+ auto_tools_called = len(
544
+ tool_parser.prev_tool_call_arr) > 0
545
+ index = len(tool_parser.prev_tool_call_arr
546
+ ) - 1 if auto_tools_called else 0
547
else:
548
index = 0
549
@@ -576,9 +578,7 @@ class OpenAIServingChat(OpenAIServing):
578
delta=delta_message,
579
logprobs=logprobs,
580
finish_reason=output.finish_reason
579
- if not (tool_parser
580
- and len(tool_parser.prev_tool_call_arr))
581
- else "tool_calls",
581
+ if not auto_tools_called else "tool_calls",
582
stop_reason=output.stop_reason)
583
chunk = ChatCompletionStreamResponse(
584
id=request_id,
@@ -680,8 +680,10 @@ class OpenAIServingChat(OpenAIServing):
680
else:
681
logprobs = None
682
683
- # by default, tools are not used.
684
- tools_called = False
683
+ # In the OpenAI API the finish_reason is "tools_called"
684
+ # if the tool choice is auto and the model produced a tool
685
+ # call. The same is not true for named function calls
686
+ auto_tools_called = False
687
688
# if auto tools are not enabled, and a named tool choice using
689
# outlines is not being used
@@ -703,7 +705,6 @@ class OpenAIServingChat(OpenAIServing):
705
name=request.tool_choice.function.name,
706
arguments=output.text))
707
])
706
- tools_called = True
708
709
# if the request doesn't use tool choice
710
# OR specifies to not use a tool
@@ -725,7 +726,10 @@ class OpenAIServingChat(OpenAIServing):
726
727
tool_call_info = tool_parser.extract_tool_calls(
728
output.text, request=request)
728
- tools_called = tool_call_info.tools_called
729
+ # In the OpenAI API the finish_reason is "tools_called"
730
+ # if the tool choice is auto and the model produced a tool
731
+ # call. The same is not true for named function calls
732
+ auto_tools_called = tool_call_info.tools_called
733
if tool_call_info.tools_called:
734
message = ChatMessage(role=role,
735
content=tool_call_info.content,
@@ -748,7 +752,7 @@ class OpenAIServingChat(OpenAIServing):
752
index=output.index,
753
message=message,
754
logprobs=logprobs,
751
- finish_reason="tool_calls" if tools_called else
755
+ finish_reason="tool_calls" if auto_tools_called else
756
output.finish_reason if output.finish_reason else "stop",
757
stop_reason=output.stop_reason)
758
choices.append(choice_data)