A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[gpt-oss] Small bug fixes for frontend (#22512)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
Aug 11, 2025 at 22:04 UTC
ad344ef552ece428d12e04fbcb64b8b50768283b
5 files changed
+77
-33
vllm/entrypoints/context.py
+42
-14
@@ -1,15 +1,20 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import json
4
import logging
5
from abc import ABC, abstractmethod
6
+from typing import TYPE_CHECKING, Union
7
6
-from openai_harmony import Message, Role, StreamState
8
+from openai_harmony import Author, Message, Role, StreamState, TextContent
9
10
from vllm.entrypoints.harmony_utils import (
11
get_encoding, get_streamable_parser_for_assistant, render_for_completion)
12
from vllm.entrypoints.tool import Tool
13
from vllm.outputs import RequestOutput
14
15
+if TYPE_CHECKING:
16
+ from mcp.client import ClientSession
17
+
18
logger = logging.getLogger(__name__)
19
20
@@ -71,6 +76,7 @@ class HarmonyContext(ConversationContext):
76
def append_output(self, output) -> None:
77
if isinstance(output, RequestOutput):
78
output_token_ids = output.outputs[0].token_ids
79
+ self.parser = get_streamable_parser_for_assistant()
80
for token_id in output_token_ids:
81
self.parser.process(token_id)
82
output_msgs = self.parser.messages
@@ -106,19 +112,41 @@ class HarmonyContext(ConversationContext):
112
def render_for_completion(self) -> list[int]:
113
return render_for_completion(self.messages)
114
109
- async def call_search_tool(
110
- self,
111
- tool_session: Tool,
112
- last_msg: Message,
113
- ) -> list[Message]:
114
- return await tool_session.get_result(self)
115
-
116
- async def call_python_tool(
117
- self,
118
- tool_session: Tool,
119
- last_msg: Message,
120
- ) -> list[Message]:
121
- return await tool_session.get_result(self)
115
+ async def call_search_tool(self, tool_session: Union["ClientSession",
116
+ Tool],
117
+ last_msg: Message) -> list[Message]:
118
+ if isinstance(tool_session, Tool):
119
+ return await tool_session.get_result(self)
120
+ tool_name = last_msg.recipient.split(".")[1]
121
+ args = json.loads(last_msg.content[0].text)
122
+ result = await tool_session.call_tool(tool_name, args)
123
+ result_str = result.content[0].text
124
+ content = TextContent(text=result_str)
125
+ author = Author(role=Role.TOOL, name=last_msg.recipient)
126
+ return [
127
+ Message(author=author, content=[content], recipient=Role.ASSISTANT)
128
+ ]
129
+
130
+ async def call_python_tool(self, tool_session: Union["ClientSession",
131
+ Tool],
132
+ last_msg: Message) -> list[Message]:
133
+ if isinstance(tool_session, Tool):
134
+ return await tool_session.get_result(self)
135
+ param = {
136
+ "code": last_msg.content[0].text,
137
+ }
138
+ result = await tool_session.call_tool("python", param)
139
+ result_str = result.content[0].text
140
+
141
+ content = TextContent(text=result_str)
142
+ author = Author(role=Role.TOOL, name="python")
143
+
144
+ return [
145
+ Message(author=author,
146
+ content=[content],
147
+ channel=last_msg.channel,
148
+ recipient=Role.ASSISTANT)
149
+ ]
150
151
152
class StreamingHarmonyContext(HarmonyContext):
vllm/entrypoints/openai/protocol.py
+3
-2
@@ -19,8 +19,8 @@ from openai.types.chat.chat_completion_message import (
19
# yapf: enable
20
from openai.types.responses import (ResponseFunctionToolCall,
21
ResponseInputItemParam, ResponseOutputItem,
22
- ResponsePrompt, ResponseStatus,
23
- ResponseTextConfig)
22
+ ResponsePrompt, ResponseReasoningItem,
23
+ ResponseStatus, ResponseTextConfig)
24
from openai.types.responses.response import ToolChoice
25
from openai.types.responses.tool import Tool
26
from openai.types.shared import Metadata, Reasoning
@@ -239,6 +239,7 @@ def get_logits_processors(processors: Optional[LogitsProcessors],
239
240
241
ResponseInputOutputItem: TypeAlias = Union[ResponseInputItemParam,
242
+ ResponseReasoningItem,
243
ResponseFunctionToolCall]
244
245
vllm/entrypoints/openai/serving_responses.py
+16
-13
@@ -16,8 +16,7 @@ from fastapi import Request
16
from openai import BaseModel
17
# yapf conflicts with isort for this block
18
# yapf: disable
19
-from openai.types.responses import (ResponseContentPartDoneEvent,
20
- ResponseCreatedEvent,
19
+from openai.types.responses import (ResponseCreatedEvent,
20
ResponseFunctionToolCall,
21
ResponseInProgressEvent,
22
ResponseOutputItem,
@@ -54,7 +53,7 @@ from vllm.entrypoints.openai.protocol import (ErrorResponse,
53
# yapf: enable
54
from vllm.entrypoints.openai.serving_engine import OpenAIServing
55
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
57
-from vllm.entrypoints.tool_server import ToolServer
56
+from vllm.entrypoints.tool_server import MCPToolServer, ToolServer
57
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
58
from vllm.logger import init_logger
59
from vllm.outputs import CompletionOutput
@@ -238,6 +237,15 @@ class OpenAIServingResponses(OpenAIServing):
237
if raw_request:
238
raw_request.state.request_metadata = request_metadata
239
240
+ if self.tool_server is not None and isinstance(
241
+ self.tool_server, MCPToolServer
242
+ ) and (request.background or request.stream) and request.tools and any(
243
+ tool.type in ["web_search_preview", "code_interpreter"]
244
+ for tool in request.tools):
245
+ return self.create_error_response(
246
+ "MCP tool server is not supported in background mode and "
247
+ "streaming mode")
248
+
249
# Schedule the request and get the result generator.
250
generators: list[AsyncGenerator[ConversationContext, None]] = []
251
@@ -844,9 +852,13 @@ class OpenAIServingResponses(OpenAIServing):
852
type="reasoning",
853
content=[
854
ResponseReasoningTextContent(
847
- text=previous_item.content[0].text),
855
+ text=previous_item.content[0].text,
856
+ type="reasoning_text",
857
+ ),
858
],
859
status="completed",
860
+ id=current_item_id,
861
+ summary=[],
862
)
863
yield _send_event(
864
ResponseReasoningTextDoneEvent(
@@ -857,15 +869,6 @@ class OpenAIServingResponses(OpenAIServing):
869
content_index=current_content_index,
870
text=previous_item.content[0].text,
871
))
860
- yield _send_event(
861
- ResponseContentPartDoneEvent(
862
- type="response.content_part.done",
863
- item_id=current_item_id,
864
- sequence_number=-1,
865
- output_index=current_output_index,
866
- content_index=current_content_index,
867
- part=reasoning_item,
868
- ))
872
yield _send_event(
873
ResponseOutputItemDoneEvent(
874
type="response.output_item.done",
vllm/entrypoints/tool.py
+13
-2
@@ -2,7 +2,9 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import os
4
from abc import ABC, abstractmethod
5
-from typing import TYPE_CHECKING, Any
5
+from typing import TYPE_CHECKING, Any, Optional
6
+
7
+from openai_harmony import Message
8
9
from vllm.logger import init_logger
10
@@ -70,7 +72,16 @@ class HarmonyPythonTool(Tool):
72
"gpt_oss is not installed, code interpreter is disabled")
73
return
74
73
- self.python_tool = PythonTool()
75
+ # NOTE (Chen): as of gpt-oss 0.0.2, there is a bug in _make_response
76
+ # and we do the following monkey patch to fix it.
77
+ class PatchedGptOssPythonTool(PythonTool):
78
+
79
+ def _make_response(self,
80
+ output: str,
81
+ channel: Optional[str] = None) -> Message:
82
+ return super()._make_response(output)
83
+
84
+ self.python_tool = PatchedGptOssPythonTool()
85
logger.info_once("Code interpreter tool initialized")
86
87
async def get_result(self, context: "ConversationContext") -> Any:
vllm/entrypoints/tool_server.py
+3
-2
@@ -4,7 +4,7 @@ from abc import ABC, abstractmethod
4
from contextlib import AbstractAsyncContextManager, asynccontextmanager
5
from typing import TYPE_CHECKING, Any, Optional
6
7
-from openai_harmony import ToolNamespaceConfig
7
+from openai_harmony import ToolDescription, ToolNamespaceConfig
8
9
from vllm.entrypoints.tool import HarmonyBrowserTool, HarmonyPythonTool, Tool
10
from vllm.logger import init_logger
@@ -105,7 +105,6 @@ class MCPToolServer(ToolServer):
105
self.harmony_tool_descriptions = {}
106
107
async def add_tool_server(self, server_url: str):
108
- from mcp.types import ToolDescription
108
tool_urls = server_url.split(",")
109
self.harmony_tool_descriptions = {}
110
self.urls: dict[str, str] = {}
@@ -133,6 +132,8 @@ class MCPToolServer(ToolServer):
132
logger.warning(
133
"Tool %s already exists. Ignoring duplicate tool server %s",
134
tool_from_mcp.name, url)
135
+ logger.info("MCPToolServer initialized with tools: %s",
136
+ list(self.harmony_tool_descriptions.keys()))
137
138
def has_tool(self, tool_name: str):
139
return tool_name in self.harmony_tool_descriptions