A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] [Frontend] Cleanup gpt-oss non-streaming chat tool calls (#25514)
Signed-off-by: Ben Browning <bbrownin@redhat.com>
Ben Browning committed
Sep 23, 2025 at 23:20 UTC
5caaeb714ce3fd08de9c2e87848b4825bb4b676d
4 files changed
+102
-12
tests/entrypoints/openai/test_serving_chat.py
+1
@@ -194,6 +194,7 @@ async def test_gpt_oss_multi_turn_chat(gptoss_client: OpenAI,
194
assert tc.function is not None and tc.function.name == "get_current_weather"
195
args1 = tc.function.arguments
196
assert args1 is not None and len(args1) > 0
197
+ assert not first_msg.content
198
199
messages.append({"role": "assistant", "content": args1})
200
messages.append({
tests/tool_use/test_openai_tool_parser.py
+74
-2
@@ -70,7 +70,12 @@ def test_extract_tool_calls_no_tools(openai_tool_parser, harmony_encoding):
70
assert extracted_info.content == "This is a test"
71
72
73
-def test_extract_tool_calls_single_tool(openai_tool_parser, harmony_encoding):
73
+@pytest.mark.parametrize("tool_args", [
74
+ '{"location": "Tokyo"}',
75
+ '{\n"location": "Tokyo"\n}',
76
+])
77
+def test_extract_tool_calls_single_tool(openai_tool_parser, harmony_encoding,
78
+ tool_args):
79
convo = Conversation.from_messages([
80
Message.from_role_and_content(Role.USER,
81
"What is the weather in Tokyo?"),
@@ -80,7 +85,7 @@ def test_extract_tool_calls_single_tool(openai_tool_parser, harmony_encoding):
85
).with_channel("analysis"),
86
Message.from_role_and_content(
87
Role.ASSISTANT,
83
- '{"location": "Tokyo"}').with_channel("commentary").with_recipient(
88
+ tool_args).with_channel("commentary").with_recipient(
89
"functions.get_current_weather").with_content_type("json"),
90
])
91
token_ids = harmony_encoding.render_conversation_for_completion(
@@ -121,6 +126,17 @@ def test_extract_tool_calls_multiple_tools(
126
Role.ASSISTANT,
127
'{"location": "Tokyo"}').with_channel("commentary").with_recipient(
128
"functions.get_user_location").with_content_type("json"),
129
+ Message.from_role_and_content(
130
+ Role.ASSISTANT, '{"location": "Tokyo"}').with_channel(
131
+ "commentary").with_recipient("functions.no_content_type"),
132
+ Message.from_role_and_content(Role.ASSISTANT, "foo").with_channel(
133
+ "commentary").with_recipient("functions.not_json_no_content_type"),
134
+ Message.from_role_and_content(
135
+ Role.ASSISTANT, '{}').with_channel("commentary").with_recipient(
136
+ "functions.empty_args").with_content_type("json"),
137
+ Message.from_role_and_content(
138
+ Role.ASSISTANT, '').with_channel("commentary").with_recipient(
139
+ "functions.no_args").with_content_type("json"),
140
])
141
token_ids = harmony_encoding.render_conversation_for_completion(
142
convo,
@@ -141,7 +157,63 @@ def test_extract_tool_calls_multiple_tools(
157
ToolCall(function=FunctionCall(
158
name="get_user_location",
159
arguments=json.dumps({"location": "Tokyo"}),
160
+ )),
161
+ ToolCall(function=FunctionCall(
162
+ name="no_content_type",
163
+ arguments=json.dumps({"location": "Tokyo"}),
164
+ )),
165
+ ToolCall(function=FunctionCall(
166
+ name="not_json_no_content_type",
167
+ arguments="foo",
168
+ )),
169
+ ToolCall(function=FunctionCall(
170
+ name="empty_args",
171
+ arguments=json.dumps({}),
172
+ )),
173
+ ToolCall(function=FunctionCall(
174
+ name="no_args",
175
+ arguments="",
176
))
177
]
178
assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
179
assert extracted_info.content is None
180
+
181
+
182
+def test_extract_tool_calls_with_content(
183
+ openai_tool_parser,
184
+ harmony_encoding,
185
+):
186
+ final_content = "This tool call will get the weather."
187
+ convo = Conversation.from_messages([
188
+ Message.from_role_and_content(
189
+ Role.USER, "What is the weather in Tokyo based on where I'm at?"),
190
+ Message.from_role_and_content(
191
+ Role.ASSISTANT,
192
+ 'User asks: "What is the weather in Tokyo?" based on their location. We need to use get_current_weather tool and get_user_location tool.', # noqa: E501
193
+ ).with_channel("analysis"),
194
+ Message.from_role_and_content(
195
+ Role.ASSISTANT,
196
+ '{"location": "Tokyo"}').with_channel("commentary").with_recipient(
197
+ "functions.get_current_weather").with_content_type("json"),
198
+ Message.from_role_and_content(Role.ASSISTANT,
199
+ final_content).with_channel("final"),
200
+ ])
201
+ token_ids = harmony_encoding.render_conversation_for_completion(
202
+ convo,
203
+ Role.ASSISTANT,
204
+ )
205
+
206
+ extracted_info = openai_tool_parser.extract_tool_calls(
207
+ "",
208
+ request=None,
209
+ token_ids=token_ids,
210
+ )
211
+ assert extracted_info.tools_called
212
+ expected_tool_calls = [
213
+ ToolCall(function=FunctionCall(
214
+ name="get_current_weather",
215
+ arguments=json.dumps({"location": "Tokyo"}),
216
+ )),
217
+ ]
218
+ assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
219
+ assert extracted_info.content == final_content
vllm/entrypoints/openai/serving_chat.py
+5
-8
@@ -1186,6 +1186,10 @@ class OpenAIServingChat(OpenAIServing):
1186
logprobs = None
1187
1188
if self.use_harmony:
1189
+ reasoning_content, content, _ = parse_chat_output(token_ids)
1190
+ if not request.include_reasoning:
1191
+ reasoning_content = None
1192
+
1193
if self.tool_parser is not None:
1194
tool_parser = self.tool_parser(tokenizer)
1195
# NOTE: We use token_ids for openai tool parser
@@ -1194,10 +1198,7 @@ class OpenAIServingChat(OpenAIServing):
1198
request=request,
1199
token_ids=token_ids, # type: ignore
1200
)
1197
- reasoning_content, content = None, tool_call_info.content
1198
- if request.include_reasoning:
1199
- reasoning_content, content, _ = parse_chat_output(
1200
- token_ids)
1201
+ content = tool_call_info.content
1202
message = ChatMessage(
1203
role=role,
1204
reasoning_content=reasoning_content,
@@ -1205,10 +1206,6 @@ class OpenAIServingChat(OpenAIServing):
1206
tool_calls=tool_call_info.tool_calls,
1207
)
1208
else:
1208
- reasoning_content, content, _ = parse_chat_output(
1209
- token_ids)
1210
- if not request.include_reasoning:
1211
- reasoning_content = None
1209
message = ChatMessage(
1210
role=role,
1211
reasoning_content=reasoning_content,
vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py
+22
-2
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from __future__ import annotations
4
5
+import json
6
from collections.abc import Sequence
7
from typing import TYPE_CHECKING
8
@@ -12,10 +13,13 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
13
FunctionCall, ToolCall)
14
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
15
ToolParser, ToolParserManager)
16
+from vllm.logger import init_logger
17
18
if TYPE_CHECKING:
19
from vllm.transformers_utils.tokenizer import AnyTokenizer
20
21
+logger = init_logger(__name__)
22
+
23
24
@ToolParserManager.register_module("openai")
25
class OpenAIToolParser(ToolParser):
@@ -40,17 +44,33 @@ class OpenAIToolParser(ToolParser):
44
45
if len(parser.messages) > 0:
46
for msg in parser.messages:
47
+ if len(msg.content) < 1:
48
+ continue
49
+ msg_text = msg.content[0].text
50
if msg.recipient and msg.recipient.startswith("functions."):
51
+ # If no content-type is given assume JSON, as that's the
52
+ # most common case with gpt-oss models.
53
+ if not msg.content_type or "json" in msg.content_type:
54
+ # load and dump the JSON text to check validity and
55
+ # remove any extra newlines or other odd formatting
56
+ try:
57
+ tool_args = json.dumps(json.loads(msg_text))
58
+ except json.JSONDecodeError:
59
+ logger.exception(
60
+ "Error decoding JSON tool call from response.")
61
+ tool_args = msg_text
62
+ else:
63
+ tool_args = msg_text
64
tool_calls.append(
65
ToolCall(
66
type="function",
67
function=FunctionCall(
68
name=msg.recipient.split("functions.")[1],
49
- arguments=msg.content[0].text,
69
+ arguments=tool_args,
70
),
71
))
72
elif msg.channel == "final":
53
- final_content = msg.content[0].text
73
+ final_content = msg_text
74
75
return ExtractedToolCallInformation(
76
tools_called=len(tool_calls) > 0,