A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Bugfix] [issue-21565] Fix the incompatibility issue with stream and named function calling when Thinking is disabled (#21573)
Signed-off-by: wangzi <3220100013@zju.edu.cn> Co-authored-by: wangzi <3220100013@zju.edu.cn>
Hongsheng Liu committed
Jul 28, 2025 at 13:43 UTC
7656cf4cf32b122cb0bf59a17b97a30e777065e0
2 files changed
+23
-7
tests/entrypoints/openai/test_completion_with_function_calling.py
+11
-2
@@ -1,6 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from typing import Union
5
+
6
import openai # use the official client for correctness check
7
import pytest
8
import pytest_asyncio
@@ -40,10 +42,17 @@ async def client(server):
42
@pytest.mark.asyncio
43
@pytest.mark.parametrize("model_name", [MODEL_NAME])
44
@pytest.mark.parametrize("stream", [True, False])
43
-@pytest.mark.parametrize("tool_choice", ["auto", "required"])
45
+@pytest.mark.parametrize("tool_choice", [
46
+ "auto", "required", {
47
+ "type": "function",
48
+ "function": {
49
+ "name": "get_current_weather"
50
+ }
51
+ }
52
+])
53
@pytest.mark.parametrize("enable_thinking", [True, False])
54
async def test_function_tool_use(client: openai.AsyncOpenAI, model_name: str,
46
- stream: bool, tool_choice: str,
55
+ stream: bool, tool_choice: Union[str, dict],
56
enable_thinking: bool):
57
tools = [
58
{
vllm/entrypoints/openai/serving_chat.py
+12
-5
@@ -623,7 +623,7 @@ class OpenAIServingChat(OpenAIServing):
623
624
# handle streaming deltas for tools with named tool_choice
625
if tool_choice_function_name:
626
- if (self.reasoning_parser
626
+ if (self.reasoning_parser and not reasoning_end_arr[i]
627
and not reasoning_parser.is_reasoning_end(
628
previous_token_ids)):
629
assert reasoning_parser is not None
@@ -637,11 +637,18 @@ class OpenAIServingChat(OpenAIServing):
637
current_token_ids,
638
output.token_ids,
639
))
640
- # When encountering think end id in delta_token_ids,
641
- # process the `content`. Only keep 'content',
642
- # remove 'reasoning_content'
640
+ # When encountering think end id in delta_token_ids
641
+ # or think end id in prompt_token_ids
642
+ # i.e {"enable_thinking": False},
643
+ # set reasoning status to end.
644
+ # Only keep 'content', remove 'reasoning_content'.
645
if reasoning_parser.is_reasoning_end(
644
- list(output.token_ids)):
646
+ list(output.token_ids)) or \
647
+ (res.prompt_token_ids and
648
+ reasoning_parser.is_reasoning_end(
649
+ list(res.prompt_token_ids)
650
+ )):
651
+ reasoning_end_arr[i] = True
652
if delta_message and delta_message.content:
653
# This need to be added to next `delta_text`
654
current_text = delta_message.content