A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 23:45 UTC
[responsesAPI][5] ResponsesParser with tools for full MCP python loop (#29798)
Signed-off-by: Andrew Xia <axia@fb.com> Signed-off-by: Andrew Xia <axia@meta.com> Co-authored-by: Andrew Xia <axia@fb.com>
Andrew Xia committed
Dec 5, 2025 at 08:11 UTC
da7bc54ea8f44a2dcacc4a9869721bd105006e10
8 files changed
+347
-16
examples/online_serving/openai_responses_client_with_tools.py
+1
-1
@@ -3,7 +3,7 @@
3
"""
4
Set up this example by starting a vLLM OpenAI-compatible server with tool call
5
options enabled.
6
-Reasoning models can be used through the Responses API as seen here
6
+Reasoning models can be used through the Responses API as seen here
7
https://platform.openai.com/docs/api-reference/responses
8
For example:
9
vllm serve Qwen/Qwen3-1.7B --reasoning-parser qwen3 \
tests/entrypoints/openai/test_response_api_parsable_context.py
+96
-3
@@ -1,6 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import importlib
5
+import json
6
7
import pytest
8
import pytest_asyncio
@@ -13,12 +15,27 @@ MODEL_NAME = "Qwen/Qwen3-8B"
15
16
@pytest.fixture(scope="module")
17
def server():
16
- args = ["--reasoning-parser", "qwen3", "--max_model_len", "5000"]
18
+ assert importlib.util.find_spec("gpt_oss") is not None, (
19
+ "Harmony tests require gpt_oss package to be installed"
20
+ )
21
+
22
+ args = [
23
+ "--reasoning-parser",
24
+ "qwen3",
25
+ "--max_model_len",
26
+ "5000",
27
+ "--structured-outputs-config.backend",
28
+ "xgrammar",
29
+ "--enable-auto-tool-choice",
30
+ "--tool-call-parser",
31
+ "hermes",
32
+ "--tool-server",
33
+ "demo",
34
+ ]
35
env_dict = dict(
36
VLLM_ENABLE_RESPONSES_API_STORE="1",
37
VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT="1",
20
- # uncomment for tool calling
21
- # PYTHON_EXECUTION_BACKEND="dangerously_use_uv",
38
+ PYTHON_EXECUTION_BACKEND="dangerously_use_uv",
39
)
40
41
with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
@@ -85,3 +102,79 @@ async def test_reasoning_and_function_items(client: OpenAI, model_name: str):
102
assert response.output[0].type == "reasoning"
103
assert response.output[1].type == "message"
104
assert type(response.output[1].content[0].text) is str
105
+
106
+
107
+def get_horoscope(sign):
108
+ return f"{sign}: Next Tuesday you will befriend a baby otter."
109
+
110
+
111
+def call_function(name, args):
112
+ if name == "get_horoscope":
113
+ return get_horoscope(**args)
114
+ else:
115
+ raise ValueError(f"Unknown function: {name}")
116
+
117
+
118
+@pytest.mark.asyncio
119
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
120
+async def test_function_call_first_turn(client: OpenAI, model_name: str):
121
+ tools = [
122
+ {
123
+ "type": "function",
124
+ "name": "get_horoscope",
125
+ "description": "Get today's horoscope for an astrological sign.",
126
+ "parameters": {
127
+ "type": "object",
128
+ "properties": {
129
+ "sign": {"type": "string"},
130
+ },
131
+ "required": ["sign"],
132
+ "additionalProperties": False,
133
+ },
134
+ "strict": True,
135
+ }
136
+ ]
137
+
138
+ response = await client.responses.create(
139
+ model=model_name,
140
+ input="What is the horoscope for Aquarius today?",
141
+ tools=tools,
142
+ temperature=0.0,
143
+ )
144
+ assert response is not None
145
+ assert response.status == "completed"
146
+ assert len(response.output) == 2
147
+ assert response.output[0].type == "reasoning"
148
+ assert response.output[1].type == "function_call"
149
+
150
+ function_call = response.output[1]
151
+ assert function_call.name == "get_horoscope"
152
+ assert function_call.call_id is not None
153
+
154
+ args = json.loads(function_call.arguments)
155
+ assert "sign" in args
156
+
157
+ # the multi turn function call is tested above in
158
+ # test_reasoning_and_function_items
159
+
160
+
161
+@pytest.mark.asyncio
162
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
163
+async def test_mcp_tool_call(client: OpenAI, model_name: str):
164
+ response = await client.responses.create(
165
+ model=model_name,
166
+ input="What is 13 * 24? Use python to calculate the result.",
167
+ tools=[{"type": "code_interpreter", "container": {"type": "auto"}}],
168
+ temperature=0.0,
169
+ )
170
+
171
+ assert response is not None
172
+ assert response.status == "completed"
173
+ assert response.output[0].type == "reasoning"
174
+ assert response.output[1].type == "mcp_call"
175
+ assert type(response.output[1].arguments) is str
176
+ assert type(response.output[1].output) is str
177
+ assert response.output[2].type == "reasoning"
178
+ # make sure the correct math is in the final output
179
+ assert response.output[3].type == "message"
180
+ assert "312" in response.output[3].content[0].text
vllm/entrypoints/context.py
+88
-4
@@ -9,10 +9,16 @@ from collections.abc import Callable
9
from contextlib import AsyncExitStack
10
from typing import TYPE_CHECKING, Union
11
12
+from openai.types.responses.response_function_tool_call_output_item import (
13
+ ResponseFunctionToolCallOutputItem,
14
+)
15
from openai.types.responses.tool import Mcp
16
from openai_harmony import Author, Message, Role, StreamState, TextContent
17
18
from vllm import envs
19
+from vllm.entrypoints.chat_utils import (
20
+ ChatTemplateContentFormatOption,
21
+)
22
from vllm.entrypoints.harmony_utils import (
23
get_encoding,
24
get_streamable_parser_for_assistant,
@@ -22,16 +28,20 @@ from vllm.entrypoints.openai.parser.responses_parser import (
28
get_responses_parser_for_simple_context,
29
)
30
from vllm.entrypoints.openai.protocol import (
31
+ FunctionCall,
32
ResponseInputOutputItem,
33
ResponseRawMessageAndToken,
34
ResponsesRequest,
35
)
36
+from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ToolParser
37
from vllm.entrypoints.responses_utils import construct_tool_dicts
38
from vllm.entrypoints.tool import Tool
39
from vllm.entrypoints.tool_server import ToolServer
40
from vllm.outputs import RequestOutput
41
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
42
+from vllm.tokenizers.protocol import TokenizerLike
43
from vllm.transformers_utils.tokenizer import AnyTokenizer
44
+from vllm.utils import random_uuid
45
46
if TYPE_CHECKING:
47
from mcp.client import ClientSession
@@ -221,6 +231,10 @@ class ParsableContext(ConversationContext):
231
tokenizer: AnyTokenizer,
232
reasoning_parser_cls: Callable[[AnyTokenizer], ReasoningParser] | None,
233
request: ResponsesRequest,
234
+ available_tools: list[str] | None,
235
+ tool_parser_cls: Callable[[TokenizerLike], ToolParser] | None,
236
+ chat_template: str | None,
237
+ chat_template_content_format: ChatTemplateContentFormatOption,
238
):
239
self.num_prompt_tokens = 0
240
self.num_output_tokens = 0
@@ -238,12 +252,19 @@ class ParsableContext(ConversationContext):
252
reasoning_parser_cls=reasoning_parser_cls,
253
response_messages=response_messages,
254
request=request,
255
+ tool_parser_cls=tool_parser_cls,
256
)
257
+ self.tool_parser_cls = tool_parser_cls
258
+ self.request = request
259
+ self.tokenizer = tokenizer
260
261
+ self.available_tools = available_tools or []
262
self._tool_sessions: dict[str, ClientSession | Tool] = {}
263
self.called_tools: set[str] = set()
264
265
self.tool_dicts = construct_tool_dicts(request.tools, request.tool_choice)
266
+ self.chat_template = chat_template
267
+ self.chat_template_content_format = chat_template_content_format
268
269
def append_output(self, output: RequestOutput) -> None:
270
self.num_prompt_tokens = len(output.prompt_token_ids or [])
@@ -252,14 +273,50 @@ class ParsableContext(ConversationContext):
273
self.parser.process(output.outputs[0])
274
275
def append_tool_output(self, output: list[ResponseInputOutputItem]) -> None:
255
- raise NotImplementedError("Should not be called.")
276
+ self.parser.response_messages.extend(output)
277
278
def need_builtin_tool_call(self) -> bool:
279
"""Return true if the last message is a MCP tool call"""
280
+ last_message = self.parser.response_messages[-1]
281
+ # TODO: figure out which tools are MCP tools
282
+ if ( # noqa: SIM103
283
+ last_message.type == "function_call"
284
+ and last_message.name in ("code_interpreter", "python")
285
+ ):
286
+ return True
287
+
288
return False
289
290
+ async def call_python_tool(
291
+ self, tool_session: Union["ClientSession", Tool], last_msg: FunctionCall
292
+ ) -> list[ResponseInputOutputItem]:
293
+ self.called_tools.add("python")
294
+ if isinstance(tool_session, Tool):
295
+ return await tool_session.get_result_parsable_context(self)
296
+ args = json.loads(last_msg.arguments)
297
+ param = {
298
+ "code": args["code"],
299
+ }
300
+ result = await tool_session.call_tool("python", param)
301
+ result_str = result.content[0].text
302
+
303
+ message = ResponseFunctionToolCallOutputItem(
304
+ id=f"fco_{random_uuid()}",
305
+ type="function_call_output",
306
+ call_id=f"call_{random_uuid()}",
307
+ output=result_str,
308
+ status="completed",
309
+ )
310
+
311
+ return [message]
312
+
313
async def call_tool(self) -> list[ResponseInputOutputItem]:
262
- raise NotImplementedError("Should not be called.")
314
+ if not self.parser.response_messages:
315
+ return []
316
+ last_msg = self.parser.response_messages[-1]
317
+ if last_msg.name == "code_interpreter":
318
+ return await self.call_python_tool(self._tool_sessions["python"], last_msg)
319
+ return []
320
321
def render_for_completion(self):
322
raise NotImplementedError("Should not be called.")
@@ -271,11 +328,38 @@ class ParsableContext(ConversationContext):
328
request_id: str,
329
mcp_tools: dict[str, Mcp],
330
):
274
- pass
331
+ if tool_server:
332
+ for tool_name in self.available_tools:
333
+ if tool_name in self._tool_sessions:
334
+ continue
335
+
336
+ tool_type = _map_tool_name_to_tool_type(tool_name)
337
+ headers = (
338
+ mcp_tools[tool_type].headers if tool_type in mcp_tools else None
339
+ )
340
+ tool_session = await exit_stack.enter_async_context(
341
+ tool_server.new_session(tool_name, request_id, headers)
342
+ )
343
+ self._tool_sessions[tool_name] = tool_session
344
+ exit_stack.push_async_exit(self.cleanup_session)
345
346
async def cleanup_session(self, *args, **kwargs) -> None:
347
"""Can be used as coro to used in __aexit__"""
278
- raise NotImplementedError("Should not be called.")
348
+
349
+ async def cleanup_tool_session(tool_session):
350
+ if not isinstance(tool_session, Tool):
351
+ logger.info(
352
+ "Cleaning up tool session for %s", tool_session._client_info
353
+ )
354
+ with contextlib.suppress(Exception):
355
+ await tool_session.call_tool("cleanup_session", {})
356
+
357
+ await asyncio.gather(
358
+ *(
359
+ cleanup_tool_session(self._tool_sessions[tool])
360
+ for tool in self.called_tools
361
+ )
362
+ )
363
364
365
class HarmonyContext(ConversationContext):
vllm/entrypoints/openai/parser/responses_parser.py
+34
@@ -3,6 +3,7 @@
3
import logging
4
from collections.abc import Callable
5
6
+from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
7
from openai.types.responses.response_output_message import ResponseOutputMessage
8
from openai.types.responses.response_output_text import ResponseOutputText
9
from openai.types.responses.response_reasoning_item import (
@@ -11,8 +12,10 @@ from openai.types.responses.response_reasoning_item import (
12
)
13
14
from vllm.entrypoints.openai.protocol import ResponseInputOutputItem, ResponsesRequest
15
+from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ToolParser
16
from vllm.outputs import CompletionOutput
17
from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
18
+from vllm.tokenizers.protocol import TokenizerLike
19
from vllm.transformers_utils.tokenizer import AnyTokenizer
20
from vllm.utils import random_uuid
21
@@ -29,6 +32,7 @@ class ResponsesParser:
32
reasoning_parser_cls: Callable[[AnyTokenizer], ReasoningParser],
33
response_messages: list[ResponseInputOutputItem],
34
request: ResponsesRequest,
35
+ tool_parser_cls: Callable[[TokenizerLike], ToolParser] | None,
36
):
37
self.response_messages: list[ResponseInputOutputItem] = (
38
# TODO: initial messages may not be properly typed
@@ -39,6 +43,9 @@ class ResponsesParser:
43
self.request = request
44
45
self.reasoning_parser_instance = reasoning_parser_cls(tokenizer)
46
+ self.tool_parser_instance = None
47
+ if tool_parser_cls is not None:
48
+ self.tool_parser_instance = tool_parser_cls(tokenizer)
49
50
def process(self, output: CompletionOutput) -> "ResponsesParser":
51
reasoning_content, content = self.reasoning_parser_instance.extract_reasoning(
@@ -59,6 +66,29 @@ class ResponsesParser:
66
)
67
)
68
69
+ function_calls: list[ResponseFunctionToolCall] = []
70
+ if self.tool_parser_instance is not None:
71
+ tool_call_info = self.tool_parser_instance.extract_tool_calls(
72
+ content if content is not None else "",
73
+ request=self.request, # type: ignore
74
+ )
75
+ if tool_call_info is not None and tool_call_info.tools_called:
76
+ # extract_tool_calls() returns a list of tool calls.
77
+ function_calls.extend(
78
+ ResponseFunctionToolCall(
79
+ id=f"fc_{random_uuid()}",
80
+ call_id=f"call_{random_uuid()}",
81
+ type="function_call",
82
+ status="completed",
83
+ name=tool_call.function.name,
84
+ arguments=tool_call.function.arguments,
85
+ )
86
+ for tool_call in tool_call_info.tool_calls
87
+ )
88
+ content = tool_call_info.content
89
+ if content and content.strip() == "":
90
+ content = None
91
+
92
if content:
93
self.response_messages.append(
94
ResponseOutputMessage(
@@ -76,6 +106,8 @@ class ResponsesParser:
106
],
107
)
108
)
109
+ if len(function_calls) > 0:
110
+ self.response_messages.extend(function_calls)
111
112
return self
113
@@ -86,6 +118,7 @@ def get_responses_parser_for_simple_context(
118
reasoning_parser_cls: Callable[[AnyTokenizer], ReasoningParser],
119
response_messages: list[ResponseInputOutputItem],
120
request: ResponsesRequest,
121
+ tool_parser_cls,
122
) -> ResponsesParser:
123
"""Factory function to create a ResponsesParser with
124
optional reasoning parser.
@@ -98,4 +131,5 @@ def get_responses_parser_for_simple_context(
131
reasoning_parser_cls=reasoning_parser_cls,
132
response_messages=response_messages,
133
request=request,
134
+ tool_parser_cls=tool_parser_cls,
135
)
vllm/entrypoints/openai/serving_engine.py
+59
-6
@@ -18,6 +18,16 @@ from pydantic import ConfigDict, TypeAdapter
18
from starlette.datastructures import Headers
19
from typing_extensions import TypeIs
20
21
+from vllm.entrypoints.context import (
22
+ HarmonyContext,
23
+ ParsableContext,
24
+ StreamingHarmonyContext,
25
+)
26
+from vllm.entrypoints.openai.protocol import (
27
+ FunctionCall,
28
+ ResponseInputOutputItem,
29
+ ResponsesRequest,
30
+)
31
from vllm.entrypoints.pooling.classify.protocol import (
32
ClassificationChatRequest,
33
ClassificationCompletionRequest,
@@ -39,6 +49,7 @@ from vllm.entrypoints.pooling.score.protocol import (
49
ScoreRequest,
50
ScoreResponse,
51
)
52
+from vllm.transformers_utils.tokenizer import AnyTokenizer
53
54
if sys.version_info >= (3, 12):
55
from typing import TypedDict
@@ -72,9 +83,7 @@ from vllm.entrypoints.openai.protocol import (
83
DetokenizeRequest,
84
ErrorInfo,
85
ErrorResponse,
75
- FunctionCall,
86
FunctionDefinition,
77
- ResponsesRequest,
87
TokenizeChatRequest,
88
TokenizeCompletionRequest,
89
TokenizeResponse,
@@ -85,6 +94,9 @@ from vllm.entrypoints.openai.protocol import (
94
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
95
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
96
from vllm.entrypoints.renderer import BaseRenderer, CompletionRenderer, RenderConfig
97
+from vllm.entrypoints.responses_utils import (
98
+ construct_input_messages,
99
+)
100
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse
101
from vllm.entrypoints.utils import _validate_truncation_size
102
from vllm.inputs.data import PromptType
@@ -1224,6 +1236,31 @@ class OpenAIServing:
1236
)
1237
return engine_request, tokenization_kwargs
1238
1239
+ async def _render_next_turn(
1240
+ self,
1241
+ request: ResponsesRequest,
1242
+ tokenizer: AnyTokenizer,
1243
+ messages: list[ResponseInputOutputItem],
1244
+ tool_dicts: list[dict[str, Any]] | None,
1245
+ tool_parser,
1246
+ chat_template: str | None,
1247
+ chat_template_content_format: ChatTemplateContentFormatOption,
1248
+ ):
1249
+ new_messages = construct_input_messages(
1250
+ request_input=messages,
1251
+ )
1252
+
1253
+ _, request_prompts, engine_prompts = await self._preprocess_chat(
1254
+ request,
1255
+ tokenizer,
1256
+ new_messages,
1257
+ tool_dicts=tool_dicts,
1258
+ tool_parser=tool_parser,
1259
+ chat_template=chat_template,
1260
+ chat_template_content_format=chat_template_content_format,
1261
+ )
1262
+ return request_prompts, engine_prompts
1263
+
1264
async def _generate_with_builtin_tools(
1265
self,
1266
request_id: str,
@@ -1286,11 +1323,27 @@ class OpenAIServing:
1323
1324
# Create inputs for the next turn.
1325
# Render the next prompt token ids.
1289
- prompt_token_ids = context.render_for_completion()
1290
- engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1291
- request_prompt = prompt_token_ids
1326
+ if isinstance(context, (HarmonyContext, StreamingHarmonyContext)):
1327
+ prompt_token_ids = context.render_for_completion()
1328
+ engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids)
1329
+ request_prompt = prompt_token_ids
1330
+ elif isinstance(context, ParsableContext):
1331
+ request_prompts, engine_prompts = await self._render_next_turn(
1332
+ context.request,
1333
+ context.tokenizer,
1334
+ context.parser.response_messages,
1335
+ context.tool_dicts,
1336
+ context.tool_parser_cls,
1337
+ context.chat_template,
1338
+ context.chat_template_content_format,
1339
+ )
1340
+ engine_prompt = engine_prompts[0]
1341
+ request_prompt = request_prompts[0]
1342
+
1343
# Update the sampling params.
1293
- sampling_params.max_tokens = self.max_model_len - len(prompt_token_ids)
1344
+ sampling_params.max_tokens = self.max_model_len - len(
1345
+ engine_prompt["prompt_token_ids"]
1346
+ )
1347
# OPTIMIZATION
1348
priority = orig_priority - 1
1349
sub_request += 1
vllm/entrypoints/openai/serving_responses.py
+5
-1
@@ -375,7 +375,7 @@ class OpenAIServingResponses(OpenAIServing):
375
generators: list[AsyncGenerator[ConversationContext, None]] = []
376
377
builtin_tool_list: list[str] = []
378
- if self.use_harmony and self.tool_server is not None:
378
+ if self.tool_server is not None:
379
if self.tool_server.has_tool("browser"):
380
builtin_tool_list.append("browser")
381
if self.tool_server.has_tool("python"):
@@ -423,6 +423,10 @@ class OpenAIServingResponses(OpenAIServing):
423
tokenizer=tokenizer,
424
reasoning_parser_cls=self.reasoning_parser,
425
request=request,
426
+ tool_parser_cls=self.tool_parser,
427
+ available_tools=available_tools,
428
+ chat_template=self.chat_template,
429
+ chat_template_content_format=self.chat_template_content_format,
430
)
431
else:
432
context = SimpleContext()
vllm/entrypoints/responses_utils.py
+20
-1
@@ -16,6 +16,7 @@ from openai.types.responses.response import ToolChoice
16
from openai.types.responses.response_function_tool_call_output_item import (
17
ResponseFunctionToolCallOutputItem,
18
)
19
+from openai.types.responses.response_output_item import McpCall
20
from openai.types.responses.response_output_message import ResponseOutputMessage
21
from openai.types.responses.response_reasoning_item import ResponseReasoningItem
22
from openai.types.responses.tool import Tool
@@ -25,6 +26,7 @@ from vllm.entrypoints.openai.protocol import (
26
ChatCompletionMessageParam,
27
ResponseInputOutputItem,
28
)
29
+from vllm.utils import random_uuid
30
31
32
def make_response_output_items_from_parsable_context(
@@ -36,7 +38,24 @@ def make_response_output_items_from_parsable_context(
38
if not isinstance(message, ResponseFunctionToolCallOutputItem):
39
output_messages.append(message)
40
else:
39
- raise NotImplementedError("tool calls not supported for response context")
41
+ if len(output_messages) == 0:
42
+ raise ValueError(
43
+ "Cannot have a FunctionToolCallOutput before FunctionToolCall."
44
+ )
45
+ if isinstance(output_messages[-1], ResponseFunctionToolCall):
46
+ mcp_message = McpCall(
47
+ id=f"mcp_{random_uuid()}",
48
+ arguments=output_messages[-1].arguments,
49
+ name=output_messages[-1].name,
50
+ server_label=output_messages[
51
+ -1
52
+ ].name, # TODO: store the server label
53
+ type="mcp_call",
54
+ status="completed",
55
+ output=message.output,
56
+ # TODO: support error output
57
+ )
58
+ output_messages[-1] = mcp_message
59
60
return output_messages
61
vllm/entrypoints/tool.py
+44
@@ -1,12 +1,17 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import json
4
import os
5
from abc import ABC, abstractmethod
6
from typing import TYPE_CHECKING, Any
7
8
+from openai.types.responses.response_function_tool_call_output_item import (
9
+ ResponseFunctionToolCallOutputItem,
10
+)
11
from openai_harmony import Author, Message, Role, TextContent
12
13
from vllm.logger import init_logger
14
+from vllm.utils import random_uuid
15
16
if TYPE_CHECKING:
17
# Avoid circular import.
@@ -46,6 +51,10 @@ class Tool(ABC):
51
async def get_result(self, context: "ConversationContext") -> Any:
52
pass
53
54
+ @abstractmethod
55
+ async def get_result_parsable_context(self, context: "ConversationContext") -> Any:
56
+ pass
57
+
58
59
class HarmonyBrowserTool(Tool):
60
def __init__(self):
@@ -81,6 +90,9 @@ class HarmonyBrowserTool(Tool):
90
tool_output_msgs.append(msg)
91
return tool_output_msgs
92
93
+ async def get_result_parsable_context(self, context: "ConversationContext") -> Any:
94
+ raise NotImplementedError("Not implemented yet")
95
+
96
@property
97
def tool_config(self) -> Any:
98
return self.browser_tool.tool_config
@@ -138,6 +150,38 @@ class HarmonyPythonTool(Tool):
150
tool_output_msgs.append(msg)
151
return tool_output_msgs
152
153
+ async def get_result_parsable_context(self, context: "ConversationContext") -> Any:
154
+ """
155
+ This function converts parsable context types to harmony and
156
+ back so we can use GPTOSS demo python tool
157
+ """
158
+ from vllm.entrypoints.context import ParsableContext
159
+
160
+ assert isinstance(context, ParsableContext)
161
+
162
+ last_msg = context.parser.response_messages[-1]
163
+ args = json.loads(last_msg.arguments)
164
+
165
+ last_msg_harmony = Message(
166
+ author=Author(role="assistant", name=None),
167
+ content=[TextContent(text=args["code"])],
168
+ channel="analysis",
169
+ recipient="python",
170
+ content_type="code",
171
+ )
172
+
173
+ tool_output_msgs = []
174
+ async for msg in self.python_tool.process(last_msg_harmony):
175
+ processed = ResponseFunctionToolCallOutputItem(
176
+ id=f"fco_{random_uuid()}",
177
+ type="function_call_output",
178
+ call_id=f"call_{random_uuid()}",
179
+ output=msg.content[0].text,
180
+ status="completed",
181
+ )
182
+ tool_output_msgs.append(processed)
183
+ return tool_output_msgs
184
+
185
@property
186
def tool_config(self) -> Any:
187
return self.python_tool.tool_config