A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Bugfix]: Fix the incompatibility issue with stream when Thinking is disabled (#19135)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Jun 5, 2025 at 14:24 UTC
8fc57501d3d35291c5f4ae036a6b6371f9f135b6
2 files changed
+101
-31
tests/entrypoints/openai/test_completion_with_function_calling.py
+86
-30
@@ -1,6 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from typing import NamedTuple
5
+
6
import openai # use the official client for correctness check
7
import pytest
8
import pytest_asyncio
@@ -22,7 +24,9 @@ def server(): # noqa: F811
24
"--guided-decoding-backend",
25
"xgrammar",
26
"--tool-call-parser",
25
- "hermes"
27
+ "hermes",
28
+ "--reasoning-parser",
29
+ "qwen3",
30
]
31
32
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -35,9 +39,53 @@ async def client(server):
39
yield async_client
40
41
42
+class TestCase(NamedTuple):
43
+ model_name: str
44
+ stream: bool
45
+ tool_choice: str
46
+ enable_thinking: bool
47
+
48
+
49
@pytest.mark.asyncio
39
-@pytest.mark.parametrize("model_name", [MODEL_NAME])
40
-async def test_required_tool_use(client: openai.AsyncOpenAI, model_name: str):
50
+@pytest.mark.parametrize(
51
+ "test_case",
52
+ [
53
+ TestCase(model_name=MODEL_NAME,
54
+ stream=True,
55
+ tool_choice="auto",
56
+ enable_thinking=False),
57
+ TestCase(model_name=MODEL_NAME,
58
+ stream=False,
59
+ tool_choice="auto",
60
+ enable_thinking=False),
61
+ TestCase(model_name=MODEL_NAME,
62
+ stream=True,
63
+ tool_choice="required",
64
+ enable_thinking=False),
65
+ TestCase(model_name=MODEL_NAME,
66
+ stream=False,
67
+ tool_choice="required",
68
+ enable_thinking=False),
69
+ TestCase(model_name=MODEL_NAME,
70
+ stream=True,
71
+ tool_choice="auto",
72
+ enable_thinking=True),
73
+ TestCase(model_name=MODEL_NAME,
74
+ stream=False,
75
+ tool_choice="auto",
76
+ enable_thinking=True),
77
+ TestCase(model_name=MODEL_NAME,
78
+ stream=True,
79
+ tool_choice="required",
80
+ enable_thinking=True),
81
+ TestCase(model_name=MODEL_NAME,
82
+ stream=False,
83
+ tool_choice="required",
84
+ enable_thinking=True),
85
+ ],
86
+)
87
+async def test_function_tool_use(client: openai.AsyncOpenAI,
88
+ test_case: TestCase):
89
tools = [
90
{
91
"type": "function",
@@ -126,30 +174,38 @@ async def test_required_tool_use(client: openai.AsyncOpenAI, model_name: str):
174
"forecast for the next 5 days, in fahrenheit?",
175
},
176
]
129
-
130
- # Non-streaming test
131
- chat_completion = await client.chat.completions.create(
132
- messages=messages,
133
- model=model_name,
134
- tools=tools,
135
- tool_choice="required",
136
- )
137
-
138
- assert chat_completion.choices[0].message.tool_calls is not None
139
- assert len(chat_completion.choices[0].message.tool_calls) > 0
140
-
141
- # Streaming test
142
- stream = await client.chat.completions.create(
143
- messages=messages,
144
- model=model_name,
145
- tools=tools,
146
- tool_choice="required",
147
- stream=True,
148
- )
149
-
150
- output = []
151
- async for chunk in stream:
152
- if chunk.choices and chunk.choices[0].delta.tool_calls:
153
- output.extend(chunk.choices[0].delta.tool_calls)
154
-
155
- assert len(output) > 0
177
+ if not test_case.stream:
178
+ # Non-streaming test
179
+ chat_completion = await client.chat.completions.create(
180
+ messages=messages,
181
+ model=test_case.model_name,
182
+ tools=tools,
183
+ tool_choice=test_case.tool_choice,
184
+ extra_body={
185
+ "chat_template_kwargs": {
186
+ "enable_thinking": test_case.enable_thinking
187
+ }
188
+ })
189
+
190
+ assert chat_completion.choices[0].message.tool_calls is not None
191
+ assert len(chat_completion.choices[0].message.tool_calls) > 0
192
+ else:
193
+ # Streaming test
194
+ stream = await client.chat.completions.create(
195
+ messages=messages,
196
+ model=test_case.model_name,
197
+ tools=tools,
198
+ tool_choice=test_case.tool_choice,
199
+ stream=True,
200
+ extra_body={
201
+ "chat_template_kwargs": {
202
+ "enable_thinking": test_case.enable_thinking
203
+ }
204
+ })
205
+
206
+ output = []
207
+ async for chunk in stream:
208
+ if chunk.choices and chunk.choices[0].delta.tool_calls:
209
+ output.extend(chunk.choices[0].delta.tool_calls)
210
+
211
+ assert len(output) > 0
vllm/entrypoints/openai/serving_chat.py
+15
-1
@@ -689,7 +689,21 @@ class OpenAIServingChat(OpenAIServing):
689
current_token_ids,
690
output.token_ids,
691
))
692
-
692
+ # When encountering think end id in prompt_token_ids
693
+ # i.e {"enable_thinking": False},
694
+ # set reasoning status to end.
695
+ # Remove the text and token ids related
696
+ # to 'reasoning_content'.
697
+ if res.prompt_token_ids and \
698
+ reasoning_parser.is_reasoning_end(
699
+ list(res.prompt_token_ids)):
700
+ reasoning_end_arr[i] = True
701
+ current_token_ids = list(output.token_ids)
702
+ if delta_message and delta_message.content:
703
+ current_text = delta_message.content
704
+ delta_message.content = None
705
+ else:
706
+ current_text = ""
707
# When encountering think end id in delta_token_ids,
708
# set reasoning status to end.
709
# Remove the text and token ids related