A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[gpt-oss] tool parser supports for /chat/completions [1/n] (#22386)
Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Co-authored-by: Simon Mo <simon.mo@hey.com>
Aaron Pham committed
Sep 4, 2025 at 23:39 UTC
c29fb540ff90da720490daae58bb4bfe31a91125
8 files changed
+573
-63
tests/entrypoints/openai/test_serving_chat.py
+162
-1
@@ -1,13 +1,16 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from __future__ import annotations
5
+
6
import asyncio
7
from contextlib import suppress
8
from dataclasses import dataclass, field
7
-from typing import Any, Optional
9
+from typing import TYPE_CHECKING, Any, Optional
10
from unittest.mock import MagicMock
11
12
import pytest
13
+import pytest_asyncio
14
15
from vllm.config import MultiModalConfig
16
from vllm.engine.multiprocessing.client import MQLLMEngineClient
@@ -17,6 +20,164 @@ from vllm.entrypoints.openai.serving_models import (BaseModelPath,
20
OpenAIServingModels)
21
from vllm.transformers_utils.tokenizer import get_tokenizer
22
23
+from ...utils import RemoteOpenAIServer
24
+
25
+if TYPE_CHECKING:
26
+ from openai import OpenAI
27
+
28
+GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
29
+
30
+
31
+@pytest.fixture(scope="module")
32
+def monkeypatch_module():
33
+ from _pytest.monkeypatch import MonkeyPatch
34
+ mpatch = MonkeyPatch()
35
+ yield mpatch
36
+ mpatch.undo()
37
+
38
+
39
+@pytest.fixture(scope="module")
40
+def gptoss_server(monkeypatch_module: pytest.MonkeyPatch):
41
+ with monkeypatch_module.context() as m:
42
+ m.setenv("VLLM_ATTENTION_BACKEND", "TRITON_ATTN_VLLM_V1")
43
+ args = [
44
+ "--enforce-eager",
45
+ "--max-model-len",
46
+ "8192",
47
+ "--tool-call-parser",
48
+ "openai",
49
+ "--reasoning-parser",
50
+ "openai_gptoss",
51
+ "--enable-auto-tool-choice",
52
+ ]
53
+ with RemoteOpenAIServer(GPT_OSS_MODEL_NAME, args) as remote_server:
54
+ yield remote_server
55
+
56
+
57
+@pytest_asyncio.fixture
58
+async def gptoss_client(gptoss_server):
59
+ async with gptoss_server.get_async_client() as async_client:
60
+ yield async_client
61
+
62
+
63
+@pytest.mark.asyncio
64
+async def test_gpt_oss_chat_tool_call_streaming(gptoss_client: OpenAI):
65
+ tools = [{
66
+ "type": "function",
67
+ "function": {
68
+ "name": "get_current_weather",
69
+ "description": "Get the current weather in a given location",
70
+ "parameters": {
71
+ "type": "object",
72
+ "properties": {
73
+ "city": {
74
+ "type": "string"
75
+ },
76
+ "state": {
77
+ "type": "string"
78
+ },
79
+ "unit": {
80
+ "type": "string",
81
+ "enum": ["celsius", "fahrenheit"],
82
+ },
83
+ },
84
+ "required": ["city", "state", "unit"],
85
+ },
86
+ },
87
+ }]
88
+
89
+ messages = [
90
+ {
91
+ "role": "user",
92
+ "content": "What is the weather in Dallas, TX?"
93
+ },
94
+ ]
95
+
96
+ stream = await gptoss_client.chat.completions.create(
97
+ model=GPT_OSS_MODEL_NAME, messages=messages, tools=tools, stream=True)
98
+
99
+ name = None
100
+ args_buf = ""
101
+ async for chunk in stream:
102
+ delta = chunk.choices[0].delta
103
+ if delta.tool_calls:
104
+ tc = delta.tool_calls[0]
105
+ if tc.function and tc.function.name:
106
+ name = tc.function.name
107
+ if tc.function and tc.function.arguments:
108
+ args_buf += tc.function.arguments
109
+
110
+ assert name is not None
111
+ assert len(args_buf) > 0
112
+
113
+
114
+@pytest.mark.asyncio
115
+async def test_gpt_oss_multi_turn_chat(gptoss_client: OpenAI):
116
+ tools = [{
117
+ "type": "function",
118
+ "function": {
119
+ "name": "get_current_weather",
120
+ "description": "Get the current weather in a given location",
121
+ "parameters": {
122
+ "type": "object",
123
+ "properties": {
124
+ "city": {
125
+ "type": "string"
126
+ },
127
+ "state": {
128
+ "type": "string"
129
+ },
130
+ "unit": {
131
+ "type": "string",
132
+ "enum": ["celsius", "fahrenheit"],
133
+ },
134
+ },
135
+ "required": ["city", "state", "unit"],
136
+ },
137
+ },
138
+ }]
139
+
140
+ messages = [
141
+ {
142
+ "role": "system",
143
+ "content": "you are a helpful assistant"
144
+ },
145
+ {
146
+ "role": "user",
147
+ "content": "What is the weather in Dallas, TX?"
148
+ },
149
+ ]
150
+
151
+ first = await gptoss_client.chat.completions.create(
152
+ model=GPT_OSS_MODEL_NAME,
153
+ messages=messages,
154
+ tools=tools,
155
+ temperature=0.0,
156
+ )
157
+ first_msg = first.choices[0].message
158
+ assert first_msg.tool_calls is not None and len(first_msg.tool_calls) > 0
159
+ tc = first_msg.tool_calls[0]
160
+ assert tc.function is not None and tc.function.name == "get_current_weather"
161
+ args1 = tc.function.arguments
162
+ assert args1 is not None and len(args1) > 0
163
+
164
+ messages.append({"role": "assistant", "content": args1})
165
+ messages.append({
166
+ "role": "user",
167
+ "content": "Now convert to celsius and return JSON only"
168
+ })
169
+
170
+ second = await gptoss_client.chat.completions.create(
171
+ model=GPT_OSS_MODEL_NAME,
172
+ messages=messages,
173
+ tools=tools,
174
+ temperature=0.0,
175
+ )
176
+ second_msg = second.choices[0].message
177
+ assert (second_msg.content is not None and len(second_msg.content) > 0) or \
178
+ (second_msg.tool_calls is not None and len(second_msg.tool_calls) > 0) # noqa: E501
179
+
180
+
181
MODEL_NAME = "openai-community/gpt2"
182
CHAT_TEMPLATE = "Dummy chat template for testing {}"
183
BASE_MODEL_PATHS = [BaseModelPath(name=MODEL_NAME, model_path=MODEL_NAME)]
tests/tool_use/test_openai_tool_parser.py
new
+147
@@ -0,0 +1,147 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import json
5
+
6
+import pytest
7
+from openai_harmony import (Conversation, DeveloperContent,
8
+ HarmonyEncodingName, Message, Role, SystemContent,
9
+ load_harmony_encoding)
10
+
11
+from vllm.entrypoints.openai.protocol import FunctionCall, ToolCall
12
+from vllm.entrypoints.openai.tool_parsers import OpenAIToolParser
13
+from vllm.transformers_utils.tokenizer import get_tokenizer
14
+
15
+MODEL = "gpt2"
16
+
17
+
18
+@pytest.fixture(scope="module")
19
+def openai_tokenizer():
20
+ # The parser does not use the tokenizer, but the constructor requires it.
21
+ return get_tokenizer(MODEL)
22
+
23
+
24
+@pytest.fixture
25
+def openai_tool_parser(openai_tokenizer):
26
+ return OpenAIToolParser(openai_tokenizer)
27
+
28
+
29
+@pytest.fixture(scope="module")
30
+def harmony_encoding():
31
+ return load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
32
+
33
+
34
+def assert_tool_calls(
35
+ actual_tool_calls: list[ToolCall],
36
+ expected_tool_calls: list[ToolCall],
37
+):
38
+ assert len(actual_tool_calls) == len(expected_tool_calls)
39
+
40
+ for actual_tool_call, expected_tool_call in zip(actual_tool_calls,
41
+ expected_tool_calls):
42
+ assert isinstance(actual_tool_call.id, str)
43
+ assert len(actual_tool_call.id) > 16 # Default from protocol.py
44
+ assert actual_tool_call.type == "function"
45
+ assert actual_tool_call.function == expected_tool_call.function
46
+
47
+
48
+def test_extract_tool_calls_no_tools(openai_tool_parser, harmony_encoding):
49
+ convo = Conversation.from_messages([
50
+ Message.from_role_and_content(
51
+ Role.SYSTEM,
52
+ SystemContent.new(),
53
+ ),
54
+ Message.from_role_and_content(
55
+ Role.DEVELOPER,
56
+ DeveloperContent.new().with_instructions("Talk like a pirate!")),
57
+ Message.from_role_and_content(Role.USER, "Arrr, how be you?"),
58
+ Message.from_role_and_content(Role.ASSISTANT,
59
+ "This is a test").with_channel("final")
60
+ ])
61
+ token_ids = harmony_encoding.render_conversation_for_completion(
62
+ convo, Role.ASSISTANT)
63
+ extracted_info = openai_tool_parser.extract_tool_calls(
64
+ "",
65
+ request=None,
66
+ token_ids=token_ids,
67
+ )
68
+ assert not extracted_info.tools_called
69
+ assert extracted_info.tool_calls == []
70
+ assert extracted_info.content == "This is a test"
71
+
72
+
73
+def test_extract_tool_calls_single_tool(openai_tool_parser, harmony_encoding):
74
+ convo = Conversation.from_messages([
75
+ Message.from_role_and_content(Role.USER,
76
+ "What is the weather in Tokyo?"),
77
+ Message.from_role_and_content(
78
+ Role.ASSISTANT,
79
+ 'User asks: "What is the weather in Tokyo?" We need to use get_current_weather tool.', # noqa: E501
80
+ ).with_channel("analysis"),
81
+ Message.from_role_and_content(
82
+ Role.ASSISTANT,
83
+ '{"location": "Tokyo"}').with_channel("commentary").with_recipient(
84
+ "functions.get_current_weather").with_content_type("json"),
85
+ ])
86
+ token_ids = harmony_encoding.render_conversation_for_completion(
87
+ convo, Role.ASSISTANT)
88
+
89
+ extracted_info = openai_tool_parser.extract_tool_calls(
90
+ "",
91
+ request=None,
92
+ token_ids=token_ids,
93
+ )
94
+ assert extracted_info.tools_called
95
+ expected_tool_calls = [
96
+ ToolCall(function=FunctionCall(
97
+ name="get_current_weather",
98
+ arguments=json.dumps({"location": "Tokyo"}),
99
+ ))
100
+ ]
101
+ assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
102
+ assert extracted_info.content is None
103
+
104
+
105
+def test_extract_tool_calls_multiple_tools(
106
+ openai_tool_parser,
107
+ harmony_encoding,
108
+):
109
+ convo = Conversation.from_messages([
110
+ Message.from_role_and_content(
111
+ Role.USER, "What is the weather in Tokyo based on where I'm at?"),
112
+ Message.from_role_and_content(
113
+ Role.ASSISTANT,
114
+ 'User asks: "What is the weather in Tokyo?" based on their location. We need to use get_current_weather tool and get_user_location tool.', # noqa: E501
115
+ ).with_channel("analysis"),
116
+ Message.from_role_and_content(
117
+ Role.ASSISTANT,
118
+ '{"location": "Tokyo"}').with_channel("commentary").with_recipient(
119
+ "functions.get_current_weather").with_content_type("json"),
120
+ Message.from_role_and_content(
121
+ Role.ASSISTANT,
122
+ '{"location": "Tokyo"}').with_channel("commentary").with_recipient(
123
+ "functions.get_user_location").with_content_type("json"),
124
+ ])
125
+ token_ids = harmony_encoding.render_conversation_for_completion(
126
+ convo,
127
+ Role.ASSISTANT,
128
+ )
129
+
130
+ extracted_info = openai_tool_parser.extract_tool_calls(
131
+ "",
132
+ request=None,
133
+ token_ids=token_ids,
134
+ )
135
+ assert extracted_info.tools_called
136
+ expected_tool_calls = [
137
+ ToolCall(function=FunctionCall(
138
+ name="get_current_weather",
139
+ arguments=json.dumps({"location": "Tokyo"}),
140
+ )),
141
+ ToolCall(function=FunctionCall(
142
+ name="get_user_location",
143
+ arguments=json.dumps({"location": "Tokyo"}),
144
+ ))
145
+ ]
146
+ assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
147
+ assert extracted_info.content is None
vllm/entrypoints/harmony_utils.py
+59
-13
@@ -1,5 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from __future__ import annotations
5
+
6
import datetime
7
import json
8
from collections.abc import Iterable, Sequence
@@ -18,7 +21,8 @@ from openai_harmony import (Author, Conversation, DeveloperContent,
21
Role, StreamableParser, SystemContent, TextContent,
22
ToolDescription, load_harmony_encoding)
23
21
-from vllm.entrypoints.openai.protocol import ResponseInputOutputItem
24
+from vllm.entrypoints.openai.protocol import (ChatCompletionToolsParam,
25
+ ResponseInputOutputItem)
26
from vllm.utils import random_uuid
27
28
REASONING_EFFORT = {
@@ -63,13 +67,29 @@ def get_system_message(
67
return sys_msg
68
69
66
-def get_developer_message(instructions: Optional[str] = None,
67
- tools: Optional[list[Tool]] = None) -> Message:
70
+def create_tool_definition(tool: Union[ChatCompletionToolsParam, Tool]):
71
+ if isinstance(tool, ChatCompletionToolsParam):
72
+ return ToolDescription.new(
73
+ name=tool.function.name,
74
+ description=tool.function.description,
75
+ parameters=tool.function.parameters,
76
+ )
77
+ return ToolDescription.new(
78
+ name=tool.name,
79
+ description=tool.description,
80
+ parameters=tool.parameters,
81
+ )
82
+
83
+
84
+def get_developer_message(
85
+ instructions: Optional[str] = None,
86
+ tools: Optional[list[Union[Tool, ChatCompletionToolsParam]]] = None,
87
+) -> Message:
88
dev_msg_content = DeveloperContent.new()
89
if instructions is not None:
90
dev_msg_content = dev_msg_content.with_instructions(instructions)
91
if tools is not None:
72
- function_tools = []
92
+ function_tools: list[Union[Tool, ChatCompletionToolsParam]] = []
93
for tool in tools:
94
if tool.type in ("web_search_preview", "code_interpreter"):
95
# These are built-in tools that are added to the system message.
@@ -80,11 +100,7 @@ def get_developer_message(instructions: Optional[str] = None,
100
raise ValueError(f"tool type {tool.type} not supported")
101
if function_tools:
102
function_tool_descriptions = [
83
- ToolDescription.new(
84
- name=tool.name,
85
- description=tool.description,
86
- parameters=tool.parameters,
87
- ) for tool in function_tools
103
+ create_tool_definition(tool) for tool in function_tools
104
]
105
dev_msg_content = dev_msg_content.with_function_tools(
106
function_tool_descriptions)
@@ -148,16 +164,46 @@ def parse_response_input(
164
return msg
165
166
151
-def parse_chat_input(chat_msg) -> Message:
152
- role = chat_msg["role"]
153
- content = chat_msg["content"]
167
+def parse_chat_input(chat_msg) -> list[Message]:
168
+ if not isinstance(chat_msg, dict):
169
+ # Handle Pydantic models
170
+ chat_msg = chat_msg.model_dump(exclude_none=True)
171
+
172
+ role = chat_msg.get("role")
173
+
174
+ # Assistant message with tool calls
175
+ tool_calls = chat_msg.get("tool_calls")
176
+ if role == "assistant" and tool_calls:
177
+ msgs: list[Message] = []
178
+ for call in tool_calls:
179
+ func = call.get("function", {})
180
+ name = func.get("name", "")
181
+ arguments = func.get("arguments", "") or ""
182
+ msg = Message.from_role_and_content(Role.ASSISTANT, arguments)
183
+ msg = msg.with_channel("commentary")
184
+ msg = msg.with_recipient(f"functions.{name}")
185
+ msg = msg.with_content_type("json")
186
+ msgs.append(msg)
187
+ return msgs
188
+
189
+ # Tool role message (tool output)
190
+ if role == "tool":
191
+ name = chat_msg.get("name", "")
192
+ content = chat_msg.get("content", "") or ""
193
+ msg = Message.from_author_and_content(
194
+ Author.new(Role.TOOL, f"functions.{name}"),
195
+ content).with_channel("commentary")
196
+ return [msg]
197
+
198
+ # Default: user/assistant/system messages with content
199
+ content = chat_msg.get("content", "")
200
if isinstance(content, str):
201
contents = [TextContent(text=content)]
202
else:
203
# TODO: Support refusal.
204
contents = [TextContent(text=c.get("text", "")) for c in content]
205
msg = Message.from_role_and_contents(role, contents)
160
- return msg
206
+ return [msg]
207
208
209
def render_for_completion(messages: list[Message]) -> list[int]:
vllm/entrypoints/openai/serving_chat.py
+95
-37
@@ -6,7 +6,7 @@ import json
6
import time
7
from collections.abc import AsyncGenerator, AsyncIterator
8
from collections.abc import Sequence as GenericSequence
9
-from typing import Callable, Final, Optional, Union
9
+from typing import TYPE_CHECKING, Callable, Final, Optional, Union
10
11
import jinja2
12
import partial_json_parser
@@ -489,6 +489,8 @@ class OpenAIServingChat(OpenAIServing):
489
get_streamable_parser_for_assistant()
490
for _ in range(num_choices)
491
]
492
+ harmony_tools_streamed = [False] * num_choices
493
+ tools_streamed = [False] * num_choices
494
495
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
496
tool_choice_function_name = request.tool_choice.function.name
@@ -662,13 +664,11 @@ class OpenAIServingChat(OpenAIServing):
664
665
if self.use_harmony:
666
harmony_parser = harmony_parsers[i]
667
+ prev_recipient = harmony_parser.current_recipient
668
for token_id in output.token_ids:
669
harmony_parser.process(token_id)
667
- is_reasoning = \
668
- harmony_parser.current_channel == "analysis"
669
- if not request.include_reasoning and is_reasoning:
670
- # Skip the reasoning content.
671
- continue
670
+ cur_channel = harmony_parser.current_channel
671
+ cur_recipient = harmony_parser.current_recipient
672
delta_text = harmony_parser.last_content_delta or ""
673
else:
674
delta_text = output.text
@@ -681,8 +681,7 @@ class OpenAIServingChat(OpenAIServing):
681
delta_message: Optional[DeltaMessage]
682
683
# just update previous_texts and previous_token_ids
684
- if ((tool_choice_auto or self.reasoning_parser)
685
- and not self.use_harmony):
684
+ if tool_choice_auto or self.reasoning_parser:
685
assert previous_texts is not None
686
assert all_previous_token_ids is not None
687
previous_text = previous_texts[i]
@@ -696,11 +695,54 @@ class OpenAIServingChat(OpenAIServing):
695
current_token_ids = as_list(output.token_ids)
696
697
if self.use_harmony:
699
- if is_reasoning:
700
- delta_message = DeltaMessage(
701
- reasoning_content=delta_text)
702
- else:
698
+ if cur_channel == "final":
699
delta_message = DeltaMessage(content=delta_text)
700
+ elif cur_channel == "analysis":
701
+ if request.include_reasoning:
702
+ delta_message = DeltaMessage(
703
+ reasoning_content=delta_text)
704
+ else:
705
+ delta_message = None
706
+ elif (cur_channel == "commentary" and cur_recipient
707
+ and cur_recipient.startswith("functions.")):
708
+ # Count completed tool calls to determine index
709
+ base_index = 0
710
+ for msg in harmony_parser.messages:
711
+ if (msg.channel == "commentary"
712
+ and msg.recipient
713
+ and msg.recipient.startswith(
714
+ "functions.")):
715
+ base_index += 1
716
+
717
+ if prev_recipient != cur_recipient:
718
+ tool_name = cur_recipient.split(
719
+ "functions.", 1)[1]
720
+ delta_message = DeltaMessage(tool_calls=[
721
+ DeltaToolCall(
722
+ id=make_tool_call_id(),
723
+ type="function",
724
+ function=DeltaFunctionCall(
725
+ name=tool_name,
726
+ arguments="",
727
+ ),
728
+ index=base_index,
729
+ )
730
+ ])
731
+ elif delta_text:
732
+ delta_message = DeltaMessage(tool_calls=[
733
+ DeltaToolCall(
734
+ index=base_index,
735
+ function=DeltaFunctionCall(
736
+ arguments=delta_text),
737
+ )
738
+ ])
739
+ else:
740
+ delta_message = None
741
+
742
+ if delta_message is not None:
743
+ harmony_tools_streamed[i] = True
744
+ else:
745
+ delta_message = None
746
# handle streaming deltas for tools with named tool_choice
747
elif tool_choice_function_name:
748
if (self.reasoning_parser and not reasoning_end_arr[i]
@@ -758,6 +800,7 @@ class OpenAIServingChat(OpenAIServing):
800
delta_message = DeltaMessage(tool_calls=[
801
delta_tool_call,
802
])
803
+ tools_streamed[i] = True
804
805
elif request.tool_choice == "required":
806
assert previous_texts is not None
@@ -783,6 +826,7 @@ class OpenAIServingChat(OpenAIServing):
826
if (delta_message and delta_message.tool_calls and
827
delta_message.tool_calls[0].id is not None):
828
history_tool_call_cnt += 1
829
+ tools_streamed[i] = True
830
831
# update the previous values for the next iteration
832
previous_texts[i] = current_text
@@ -859,6 +903,8 @@ class OpenAIServingChat(OpenAIServing):
903
current_token_ids=current_token_ids,
904
delta_token_ids=delta_token_ids,
905
request=request))
906
+ if delta_message and delta_message.tool_calls:
907
+ tools_streamed[i] = True
908
# when only tool calls
909
elif tool_choice_auto:
910
assert tool_parser is not None
@@ -871,6 +917,8 @@ class OpenAIServingChat(OpenAIServing):
917
current_token_ids=current_token_ids,
918
delta_token_ids=output.token_ids,
919
request=request))
920
+ if delta_message and delta_message.tool_calls:
921
+ tools_streamed[i] = True
922
923
# when only reasoning
924
elif self.reasoning_parser:
@@ -907,7 +955,10 @@ class OpenAIServingChat(OpenAIServing):
955
# wasn't ready to send a token, then
956
# get the next token without streaming a chunk
957
if delta_message is None:
910
- continue
958
+ if output.finish_reason is None:
959
+ continue
960
+ else:
961
+ delta_message = DeltaMessage()
962
963
# Log streaming delta if output logging is enabled
964
if self.enable_log_outputs and self.request_logger:
@@ -993,12 +1044,18 @@ class OpenAIServingChat(OpenAIServing):
1044
])
1045
1046
# Send the finish response for each request.n only once
1047
+ if auto_tools_called or tools_streamed[i] or (
1048
+ self.use_harmony
1049
+ and harmony_tools_streamed[i]):
1050
+ finish_reason_ = "tool_calls"
1051
+ else:
1052
+ finish_reason_ = output.finish_reason \
1053
+ if output.finish_reason else "stop"
1054
choice_data = ChatCompletionResponseStreamChoice(
1055
index=i,
1056
delta=delta_message,
1057
logprobs=logprobs,
1000
- finish_reason=output.finish_reason
1001
- if not auto_tools_called else "tool_calls",
1058
+ finish_reason=finish_reason_,
1059
stop_reason=output.stop_reason,
1060
token_ids=(as_list(output.token_ids)
1061
if request.return_token_ids else None))
@@ -1131,31 +1188,32 @@ class OpenAIServingChat(OpenAIServing):
1188
logprobs = None
1189
1190
if self.use_harmony:
1134
- reasoning_content, final_content, is_tool_call = (
1135
- parse_chat_output(token_ids))
1136
- if not request.include_reasoning:
1137
- reasoning_content = None
1138
-
1139
- if is_tool_call:
1140
- # TODO(woosuk): Implement tool call for gpt-oss.
1141
- # For now, only Responses API supports tool call for
1142
- # gpt-oss.
1143
- raise NotImplementedError(
1144
- "Tool call in Chat Completion API is not supported "
1145
- "for gpt-oss yet. Please use Responses API instead.")
1146
- else:
1147
- # Normal message
1148
- message = ChatMessage(
1149
- role=role,
1150
- reasoning_content=reasoning_content,
1151
- content=final_content,
1152
- )
1191
+ if TYPE_CHECKING:
1192
+ assert self.tool_parser is not None
1193
+ tool_parser = self.tool_parser(tokenizer)
1194
+ # NOTE: We use token_ids for openai tool parser
1195
+ tool_call_info = tool_parser.extract_tool_calls(
1196
+ "",
1197
+ request=request,
1198
+ token_ids=token_ids, # type: ignore
1199
+ )
1200
+ reasoning_content, content = None, tool_call_info.content
1201
+ if request.include_reasoning:
1202
+ reasoning_content, content, _ = parse_chat_output(
1203
+ token_ids)
1204
+ message = ChatMessage(
1205
+ role=role,
1206
+ reasoning_content=reasoning_content,
1207
+ content=content,
1208
+ tool_calls=tool_call_info.tool_calls,
1209
+ )
1210
1211
choice_data = ChatCompletionResponseChoice(
1212
index=output.index,
1213
message=message,
1214
logprobs=logprobs,
1158
- finish_reason="tool_calls" if is_tool_call else
1215
+ finish_reason="tool_calls"
1216
+ if tool_call_info.tools_called else
1217
output.finish_reason if output.finish_reason else "stop",
1218
stop_reason=output.stop_reason,
1219
)
@@ -1504,12 +1562,12 @@ class OpenAIServingChat(OpenAIServing):
1562
messages.append(sys_msg)
1563
1564
# Add developer message.
1507
- dev_msg = get_developer_message()
1565
+ dev_msg = get_developer_message(tools=request.tools)
1566
messages.append(dev_msg)
1567
1568
# Add user message.
1569
for chat_msg in request.messages:
1512
- messages.append(parse_chat_input(chat_msg))
1570
+ messages.extend(parse_chat_input(chat_msg))
1571
1572
# Render prompt token ids.
1573
prompt_token_ids = render_for_completion(messages)
vllm/entrypoints/openai/tool_parsers/__init__.py
+2
@@ -16,6 +16,7 @@ from .llama4_pythonic_tool_parser import Llama4PythonicToolParser
16
from .llama_tool_parser import Llama3JsonToolParser
17
from .minimax_tool_parser import MinimaxToolParser
18
from .mistral_tool_parser import MistralToolParser
19
+from .openai_tool_parser import OpenAIToolParser
20
from .phi4mini_tool_parser import Phi4MiniJsonToolParser
21
from .pythonic_tool_parser import PythonicToolParser
22
from .qwen3coder_tool_parser import Qwen3CoderToolParser
@@ -46,4 +47,5 @@ __all__ = [
47
"Qwen3CoderToolParser",
48
"SeedOssToolParser",
49
"Step3ToolParser",
50
+ "OpenAIToolParser",
51
]
vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py
new
+73
@@ -0,0 +1,73 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from __future__ import annotations
4
+
5
+from collections.abc import Sequence
6
+from typing import TYPE_CHECKING
7
+
8
+from vllm.entrypoints.harmony_utils import parse_output_into_messages
9
+from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
10
+ DeltaMessage,
11
+ ExtractedToolCallInformation,
12
+ FunctionCall, ToolCall)
13
+from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
14
+ ToolParser, ToolParserManager)
15
+
16
+if TYPE_CHECKING:
17
+ from vllm.transformers_utils.tokenizer import AnyTokenizer
18
+
19
+
20
+@ToolParserManager.register_module("openai")
21
+class OpenAIToolParser(ToolParser):
22
+
23
+ def __init__(self, tokenizer: AnyTokenizer):
24
+ super().__init__(tokenizer)
25
+
26
+ def extract_tool_calls(
27
+ self,
28
+ model_output: str,
29
+ request: ChatCompletionRequest,
30
+ token_ids: Sequence[int] | None = None,
31
+ ) -> ExtractedToolCallInformation:
32
+ if token_ids is None:
33
+ raise NotImplementedError(
34
+ "OpenAIToolParser requires token IDs and does not support text-based extraction." # noqa: E501
35
+ )
36
+
37
+ parser = parse_output_into_messages(token_ids)
38
+ tool_calls = []
39
+ final_content = None
40
+
41
+ if len(parser.messages) > 0:
42
+ for msg in parser.messages:
43
+ if msg.recipient and msg.recipient.startswith("functions."):
44
+ tool_calls.append(
45
+ ToolCall(
46
+ type="function",
47
+ function=FunctionCall(
48
+ name=msg.recipient.split("functions.")[1],
49
+ arguments=msg.content[0].text,
50
+ ),
51
+ ))
52
+ elif msg.channel == "final":
53
+ final_content = msg.content[0].text
54
+
55
+ return ExtractedToolCallInformation(
56
+ tools_called=len(tool_calls) > 0,
57
+ tool_calls=tool_calls,
58
+ content=final_content,
59
+ )
60
+
61
+ def extract_tool_calls_streaming(
62
+ self,
63
+ previous_text: str,
64
+ current_text: str,
65
+ delta_text: str,
66
+ previous_token_ids: Sequence[int],
67
+ current_token_ids: Sequence[int],
68
+ delta_token_ids: Sequence[int],
69
+ request: ChatCompletionRequest,
70
+ ) -> DeltaMessage | None:
71
+ raise NotImplementedError(
72
+ "Not being used, manual parsing in serving_chat.py" # noqa: E501
73
+ )
vllm/model_executor/models/config.py
+1
-1
@@ -256,7 +256,7 @@ class GptOssForCausalLMConfig(VerifyAndUpdateConfig):
256
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
257
decoding_config = vllm_config.decoding_config
258
if decoding_config.reasoning_backend == "":
259
- decoding_config.reasoning_backend = "GptOss"
259
+ decoding_config.reasoning_backend = "openai_gptoss"
260
261
# Increase the max capture size from 512 to 1024 for performance.
262
# NOTE(woosuk): This will increase the number of CUDA graphs
vllm/reasoning/gptoss_reasoning_parser.py
+34
-11
@@ -6,6 +6,7 @@ from typing import Optional, Union
6
7
from transformers import PreTrainedTokenizerBase
8
9
+from vllm.entrypoints.harmony_utils import parse_chat_output
10
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
11
DeltaMessage)
12
from vllm.logger import init_logger
@@ -14,7 +15,7 @@ from vllm.reasoning import ReasoningParser, ReasoningParserManager
15
logger = init_logger(__name__)
16
17
17
-@ReasoningParserManager.register_module("GptOss")
18
+@ReasoningParserManager.register_module("openai_gptoss")
19
class GptOssReasoningParser(ReasoningParser):
20
"""
21
Reasoning parser for GptOss model.
@@ -39,9 +40,10 @@ class GptOssReasoningParser(ReasoningParser):
40
return False
41
42
def extract_content_ids(self, input_ids: list[int]) -> list[int]:
42
- raise RuntimeError(
43
- "GptOss model uses harmony to extract reasoning content. This "
44
- "function should not be called.")
43
+ _, content, _ = parse_chat_output(input_ids)
44
+ if content is None:
45
+ return []
46
+ return self.model_tokenizer.encode(content)
47
48
def extract_reasoning_content_streaming(
49
self,
@@ -52,13 +54,34 @@ class GptOssReasoningParser(ReasoningParser):
54
current_token_ids: Sequence[int],
55
delta_token_ids: Sequence[int],
56
) -> Union[DeltaMessage, None]:
55
- raise RuntimeError(
56
- "GptOss model uses harmony to extract reasoning content. This "
57
- "function should not be called.")
57
+ prev_reasoning, prev_content, _ = parse_chat_output(
58
+ list(previous_token_ids))
59
+ cur_reasoning, cur_content, _ = parse_chat_output(
60
+ list(current_token_ids))
61
+ reasoning_delta = None
62
+ content_delta = None
63
+ if cur_reasoning is not None:
64
+ prev_r = prev_reasoning or ""
65
+ if cur_reasoning.startswith(prev_r):
66
+ reasoning_delta = cur_reasoning[len(prev_r):] or None
67
+ else:
68
+ reasoning_delta = cur_reasoning
69
+ if cur_content is not None:
70
+ prev_c = prev_content or ""
71
+ if cur_content.startswith(prev_c):
72
+ content_delta = cur_content[len(prev_c):] or None
73
+ else:
74
+ content_delta = cur_content
75
+ if reasoning_delta is None and content_delta is None:
76
+ return None
77
+ return DeltaMessage(reasoning_content=reasoning_delta,
78
+ content=content_delta)
79
80
def extract_reasoning_content(
60
- self, model_output: str, request: ChatCompletionRequest
81
+ self,
82
+ model_output: str,
83
+ request: ChatCompletionRequest,
84
) -> tuple[Optional[str], Optional[str]]:
62
- raise RuntimeError(
63
- "GptOss model uses harmony to extract reasoning content. This "
64
- "function should not be called.")
85
+ raise NotImplementedError(
86
+ "gpt-oss has a special branch for parsing reasoning in non-streaming mode. This method shouldn't be used." # noqa: E501
87
+ )