@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ec10cb8511
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[BugFix] Fix tool call finish reason in streaming case (#9209)

Signed-off-by: Max de Bayser <mbayser@br.ibm.com>

Maximilien de Bayser committed Oct 11, 2024 at 22:24 UTC ec10cb8511b7e30b8ff86caab2e4272ff3ceddca
1 file changed +15 -11
vllm/entrypoints/openai/serving_chat.py
+15 -11
@@ -538,10 +538,12 @@ class OpenAIServingChat(OpenAIServing):
538 # any tokens that were generated but previously
539 # matched by partial json parsing
540 # only happens if we are NOT using guided decoding
541 + auto_tools_called = False
542 if tool_parser:
542 - index = len(
543 - tool_parser.prev_tool_call_arr) - 1 if len(
544 - tool_parser.prev_tool_call_arr) > 0 else 0
543 + auto_tools_called = len(
544 + tool_parser.prev_tool_call_arr) > 0
545 + index = len(tool_parser.prev_tool_call_arr
546 + ) - 1 if auto_tools_called else 0
547 else:
548 index = 0
549
@@ -576,9 +578,7 @@ class OpenAIServingChat(OpenAIServing):
578 delta=delta_message,
579 logprobs=logprobs,
580 finish_reason=output.finish_reason
579 - if not (tool_parser
580 - and len(tool_parser.prev_tool_call_arr))
581 - else "tool_calls",
581 + if not auto_tools_called else "tool_calls",
582 stop_reason=output.stop_reason)
583 chunk = ChatCompletionStreamResponse(
584 id=request_id,
@@ -680,8 +680,10 @@ class OpenAIServingChat(OpenAIServing):
680 else:
681 logprobs = None
682
683 - # by default, tools are not used.
684 - tools_called = False
683 + # In the OpenAI API the finish_reason is "tools_called"
684 + # if the tool choice is auto and the model produced a tool
685 + # call. The same is not true for named function calls
686 + auto_tools_called = False
687
688 # if auto tools are not enabled, and a named tool choice using
689 # outlines is not being used
@@ -703,7 +705,6 @@ class OpenAIServingChat(OpenAIServing):
705 name=request.tool_choice.function.name,
706 arguments=output.text))
707 ])
706 - tools_called = True
708
709 # if the request doesn't use tool choice
710 # OR specifies to not use a tool
@@ -725,7 +726,10 @@ class OpenAIServingChat(OpenAIServing):
726
727 tool_call_info = tool_parser.extract_tool_calls(
728 output.text, request=request)
728 - tools_called = tool_call_info.tools_called
729 + # In the OpenAI API the finish_reason is "tools_called"
730 + # if the tool choice is auto and the model produced a tool
731 + # call. The same is not true for named function calls
732 + auto_tools_called = tool_call_info.tools_called
733 if tool_call_info.tools_called:
734 message = ChatMessage(role=role,
735 content=tool_call_info.content,
@@ -748,7 +752,7 @@ class OpenAIServingChat(OpenAIServing):
752 index=output.index,
753 message=message,
754 logprobs=logprobs,
751 - finish_reason="tool_calls" if tools_called else
755 + finish_reason="tool_calls" if auto_tools_called else
756 output.finish_reason if output.finish_reason else "stop",
757 stop_reason=output.stop_reason)
758 choices.append(choice_data)