@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a4ec0c5595
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 10, 10:15 UTC

[Frontend] Add MCP tool streaming support to Responses API (#31761)

Signed-off-by: Daniel Salib <danielsalib@meta.com>

daniel-salib committed Jan 8, 2026 at 17:19 UTC a4ec0c559521c055519eeabddd8279c83eb4e936
3 files changed +1387 -629
tests/entrypoints/openai/test_response_api_mcp_tools.py
+285 -194
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +
5 import pytest
6 import pytest_asyncio
7 from openai import OpenAI
@@ -13,199 +14,6 @@ from ...utils import RemoteOpenAIServer
14 MODEL_NAME = "openai/gpt-oss-20b"
15
16
16 -@pytest.fixture(scope="module")
17 -def monkeypatch_module():
18 - from _pytest.monkeypatch import MonkeyPatch
19 -
20 - mpatch = MonkeyPatch()
21 - yield mpatch
22 - mpatch.undo()
23 -
24 -
25 -@pytest.fixture(scope="module")
26 -def mcp_disabled_server(monkeypatch_module: pytest.MonkeyPatch):
27 - args = ["--enforce-eager", "--tool-server", "demo"]
28 -
29 - with monkeypatch_module.context() as m:
30 - m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1")
31 - m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv")
32 - # Helps the model follow instructions better
33 - m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1")
34 - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
35 - yield remote_server
36 -
37 -
38 -@pytest.fixture(scope="function")
39 -def mcp_enabled_server(monkeypatch_module: pytest.MonkeyPatch):
40 - args = ["--enforce-eager", "--tool-server", "demo"]
41 -
42 - with monkeypatch_module.context() as m:
43 - m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1")
44 - m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv")
45 - m.setenv("VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "code_interpreter,container")
46 - # Helps the model follow instructions better
47 - m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1")
48 - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
49 - yield remote_server
50 -
51 -
52 -@pytest_asyncio.fixture
53 -async def mcp_disabled_client(mcp_disabled_server):
54 - async with mcp_disabled_server.get_async_client() as async_client:
55 - yield async_client
56 -
57 -
58 -@pytest_asyncio.fixture
59 -async def mcp_enabled_client(mcp_enabled_server):
60 - async with mcp_enabled_server.get_async_client() as async_client:
61 - yield async_client
62 -
63 -
64 -@pytest.mark.asyncio
65 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
66 -async def test_mcp_tool_env_flag_enabled(mcp_enabled_client: OpenAI, model_name: str):
67 - response = await mcp_enabled_client.responses.create(
68 - model=model_name,
69 - input=(
70 - "Execute the following code: "
71 - "import random; print(random.randint(1, 1000000))"
72 - ),
73 - instructions=(
74 - "You must use the Python tool to execute code. Never simulate execution."
75 - ),
76 - tools=[
77 - {
78 - "type": "mcp",
79 - "server_label": "code_interpreter",
80 - # URL unused for DemoToolServer
81 - "server_url": "http://localhost:8888",
82 - }
83 - ],
84 - extra_body={"enable_response_messages": True},
85 - )
86 - assert response is not None
87 - assert response.status == "completed"
88 - # Verify output messages: Tool calls and responses on analysis channel
89 - tool_call_found = False
90 - tool_response_found = False
91 - for message in response.output_messages:
92 - recipient = message.get("recipient")
93 - if recipient and recipient.startswith("python"):
94 - tool_call_found = True
95 - assert message.get("channel") == "analysis", (
96 - "Tool call should be on analysis channel"
97 - )
98 - author = message.get("author", {})
99 - if (
100 - author.get("role") == "tool"
101 - and author.get("name")
102 - and author.get("name").startswith("python")
103 - ):
104 - tool_response_found = True
105 - assert message.get("channel") == "analysis", (
106 - "Tool response should be on analysis channel"
107 - )
108 -
109 - assert tool_call_found, "Should have found at least one Python tool call"
110 - assert tool_response_found, "Should have found at least one Python tool response"
111 - for message in response.input_messages:
112 - assert message.get("author").get("role") != "developer", (
113 - "No developer messages should be present with valid mcp tool"
114 - )
115 -
116 -
117 -@pytest.mark.asyncio
118 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
119 -async def test_mcp_tool_with_allowed_tools_star(
120 - mcp_enabled_client: OpenAI, model_name: str
121 -):
122 - """Test MCP tool with allowed_tools=['*'] to select all available tools.
123 -
124 - This E2E test verifies that the "*" wildcard works end-to-end.
125 - See test_serving_responses.py for detailed unit tests of "*" normalization.
126 - """
127 - response = await mcp_enabled_client.responses.create(
128 - model=model_name,
129 - input=(
130 - "Execute the following code: "
131 - "import random; print(random.randint(1, 1000000))"
132 - ),
133 - instructions=(
134 - "You must use the Python tool to execute code. Never simulate execution."
135 - ),
136 - tools=[
137 - {
138 - "type": "mcp",
139 - "server_label": "code_interpreter",
140 - "server_url": "http://localhost:8888",
141 - # Using "*" to allow all tools from this MCP server
142 - "allowed_tools": ["*"],
143 - }
144 - ],
145 - extra_body={"enable_response_messages": True},
146 - )
147 - assert response is not None
148 - assert response.status == "completed"
149 - # Verify tool calls work with allowed_tools=["*"]
150 - tool_call_found = False
151 - for message in response.output_messages:
152 - recipient = message.get("recipient")
153 - if recipient and recipient.startswith("python"):
154 - tool_call_found = True
155 - break
156 - assert tool_call_found, "Should have found at least one Python tool call with '*'"
157 -
158 -
159 -@pytest.mark.asyncio
160 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
161 -async def test_mcp_tool_env_flag_disabled(mcp_disabled_client: OpenAI, model_name: str):
162 - response = await mcp_disabled_client.responses.create(
163 - model=model_name,
164 - input=(
165 - "Execute the following code if the tool is present: "
166 - "import random; print(random.randint(1, 1000000))"
167 - ),
168 - tools=[
169 - {
170 - "type": "mcp",
171 - "server_label": "code_interpreter",
172 - # URL unused for DemoToolServer
173 - "server_url": "http://localhost:8888",
174 - }
175 - ],
176 - extra_body={"enable_response_messages": True},
177 - )
178 - assert response is not None
179 - assert response.status == "completed"
180 - # Verify output messages: No tool calls and responses
181 - tool_call_found = False
182 - tool_response_found = False
183 - for message in response.output_messages:
184 - recipient = message.get("recipient")
185 - if recipient and recipient.startswith("python"):
186 - tool_call_found = True
187 - assert message.get("channel") == "analysis", (
188 - "Tool call should be on analysis channel"
189 - )
190 - author = message.get("author", {})
191 - if (
192 - author.get("role") == "tool"
193 - and author.get("name")
194 - and author.get("name").startswith("python")
195 - ):
196 - tool_response_found = True
197 - assert message.get("channel") == "analysis", (
198 - "Tool response should be on analysis channel"
199 - )
200 -
201 - assert not tool_call_found, "Should not have a python call"
202 - assert not tool_response_found, "Should not have a tool response"
203 - for message in response.input_messages:
204 - assert message.get("author").get("role") != "developer", (
205 - "No developer messages should be present without a valid tool"
206 - )
207 -
208 -
17 def test_get_tool_description():
18 """Test MCPToolServer.get_tool_description filtering logic.
19
@@ -249,7 +57,10 @@ def test_get_tool_description():
57 assert result.tools[1].name == "tool3"
58
59 # Single tool
252 - result = server.get_tool_description("test_server", allowed_tools=["tool2"])
60 + result = server.get_tool_description(
61 + "test_server",
62 + allowed_tools=["tool2"],
63 + )
64 assert len(result.tools) == 1
65 assert result.tools[0].name == "tool2"
66
@@ -259,3 +70,283 @@ def test_get_tool_description():
70
71 # Empty list - returns None
72 assert server.get_tool_description("test_server", allowed_tools=[]) is None
73 +
74 +
75 +class TestMCPEnabled:
76 + """Tests that require MCP tools to be enabled via environment variable."""
77 +
78 + @pytest.fixture(scope="class")
79 + def monkeypatch_class(self):
80 + from _pytest.monkeypatch import MonkeyPatch
81 +
82 + mpatch = MonkeyPatch()
83 + yield mpatch
84 + mpatch.undo()
85 +
86 + @pytest.fixture(scope="class")
87 + def mcp_enabled_server(self, monkeypatch_class: pytest.MonkeyPatch):
88 + args = ["--enforce-eager", "--tool-server", "demo"]
89 +
90 + with monkeypatch_class.context() as m:
91 + m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1")
92 + m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv")
93 + m.setenv(
94 + "VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS", "code_interpreter,container"
95 + )
96 + # Helps the model follow instructions better
97 + m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1")
98 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
99 + yield remote_server
100 +
101 + @pytest_asyncio.fixture
102 + async def mcp_enabled_client(self, mcp_enabled_server):
103 + async with mcp_enabled_server.get_async_client() as async_client:
104 + yield async_client
105 +
106 + @pytest.mark.asyncio
107 + @pytest.mark.parametrize("model_name", [MODEL_NAME])
108 + async def test_mcp_tool_env_flag_enabled(
109 + self, mcp_enabled_client: OpenAI, model_name: str
110 + ):
111 + response = await mcp_enabled_client.responses.create(
112 + model=model_name,
113 + input=(
114 + "Execute the following code: "
115 + "import random; print(random.randint(1, 1000000))"
116 + ),
117 + instructions=(
118 + "You must use the Python tool to execute code. "
119 + "Never simulate execution."
120 + ),
121 + tools=[
122 + {
123 + "type": "mcp",
124 + "server_label": "code_interpreter",
125 + # URL unused for DemoToolServer
126 + "server_url": "http://localhost:8888",
127 + }
128 + ],
129 + extra_body={"enable_response_messages": True},
130 + )
131 + assert response is not None
132 + assert response.status == "completed"
133 + # Verify output messages: Tool calls and responses on analysis channel
134 + tool_call_found = False
135 + tool_response_found = False
136 + for message in response.output_messages:
137 + recipient = message.get("recipient")
138 + if recipient and recipient.startswith("python"):
139 + tool_call_found = True
140 + assert message.get("channel") == "analysis", (
141 + "Tool call should be on analysis channel"
142 + )
143 + author = message.get("author", {})
144 + if (
145 + author.get("role") == "tool"
146 + and author.get("name")
147 + and author.get("name").startswith("python")
148 + ):
149 + tool_response_found = True
150 + assert message.get("channel") == "analysis", (
151 + "Tool response should be on analysis channel"
152 + )
153 +
154 + assert tool_call_found, "Should have found at least one Python tool call"
155 + assert tool_response_found, (
156 + "Should have found at least one Python tool response"
157 + )
158 + for message in response.input_messages:
159 + assert message.get("author").get("role") != "developer", (
160 + "No developer messages should be present with valid mcp tool"
161 + )
162 +
163 + @pytest.mark.asyncio
164 + @pytest.mark.parametrize("model_name", [MODEL_NAME])
165 + async def test_mcp_tool_with_allowed_tools_star(
166 + self, mcp_enabled_client: OpenAI, model_name: str
167 + ):
168 + """Test MCP tool with allowed_tools=['*'] to select all available
169 + tools.
170 +
171 + This E2E test verifies that the "*" wildcard works end-to-end.
172 + See test_serving_responses.py for detailed unit tests of "*"
173 + normalization.
174 + """
175 + response = await mcp_enabled_client.responses.create(
176 + model=model_name,
177 + input=(
178 + "Execute the following code: "
179 + "import random; print(random.randint(1, 1000000))"
180 + ),
181 + instructions=(
182 + "You must use the Python tool to execute code. "
183 + "Never simulate execution."
184 + ),
185 + tools=[
186 + {
187 + "type": "mcp",
188 + "server_label": "code_interpreter",
189 + "server_url": "http://localhost:8888",
190 + # Using "*" to allow all tools from this MCP server
191 + "allowed_tools": ["*"],
192 + }
193 + ],
194 + extra_body={"enable_response_messages": True},
195 + )
196 + assert response is not None
197 + assert response.status == "completed"
198 + # Verify tool calls work with allowed_tools=["*"]
199 + tool_call_found = False
200 + for message in response.output_messages:
201 + recipient = message.get("recipient")
202 + if recipient and recipient.startswith("python"):
203 + tool_call_found = True
204 + break
205 + assert tool_call_found, (
206 + "Should have found at least one Python tool call with '*'"
207 + )
208 +
209 + @pytest.mark.flaky(reruns=3)
210 + @pytest.mark.asyncio
211 + @pytest.mark.parametrize("model_name", [MODEL_NAME])
212 + async def test_mcp_tool_calling_streaming_types(
213 + self, mcp_enabled_client: OpenAI, model_name: str
214 + ):
215 + pairs_of_event_types = {
216 + "response.completed": "response.created",
217 + "response.output_item.done": "response.output_item.added",
218 + "response.content_part.done": "response.content_part.added",
219 + "response.output_text.done": "response.output_text.delta",
220 + "response.reasoning_text.done": "response.reasoning_text.delta",
221 + "response.reasoning_part.done": "response.reasoning_part.added",
222 + "response.mcp_call_arguments.done": ("response.mcp_call_arguments.delta"),
223 + "response.mcp_call.completed": "response.mcp_call.in_progress",
224 + }
225 +
226 + tools = [
227 + {
228 + "type": "mcp",
229 + "server_label": "code_interpreter",
230 + }
231 + ]
232 + input_text = "What is 13 * 24? Use python to calculate the result."
233 +
234 + stream_response = await mcp_enabled_client.responses.create(
235 + model=model_name,
236 + input=input_text,
237 + tools=tools,
238 + stream=True,
239 + instructions=(
240 + "You must use the Python tool to execute code. "
241 + "Never simulate execution."
242 + ),
243 + )
244 +
245 + stack_of_event_types = []
246 + saw_mcp_type = False
247 + async for event in stream_response:
248 + if event.type == "response.created":
249 + stack_of_event_types.append(event.type)
250 + elif event.type == "response.completed":
251 + assert stack_of_event_types[-1] == pairs_of_event_types[event.type]
252 + stack_of_event_types.pop()
253 + elif (
254 + event.type.endswith("added")
255 + or event.type == "response.mcp_call.in_progress"
256 + ):
257 + stack_of_event_types.append(event.type)
258 + elif event.type.endswith("delta"):
259 + if stack_of_event_types[-1] == event.type:
260 + continue
261 + stack_of_event_types.append(event.type)
262 + elif (
263 + event.type.endswith("done")
264 + or event.type == "response.mcp_call.completed"
265 + ):
266 + assert stack_of_event_types[-1] == pairs_of_event_types[event.type]
267 + if "mcp_call" in event.type:
268 + saw_mcp_type = True
269 + stack_of_event_types.pop()
270 +
271 + assert len(stack_of_event_types) == 0
272 + assert saw_mcp_type, "Should have seen at least one mcp call"
273 +
274 +
275 +class TestMCPDisabled:
276 + """Tests that verify behavior when MCP tools are disabled."""
277 +
278 + @pytest.fixture(scope="class")
279 + def monkeypatch_class(self):
280 + from _pytest.monkeypatch import MonkeyPatch
281 +
282 + mpatch = MonkeyPatch()
283 + yield mpatch
284 + mpatch.undo()
285 +
286 + @pytest.fixture(scope="class")
287 + def mcp_disabled_server(self, monkeypatch_class: pytest.MonkeyPatch):
288 + args = ["--enforce-eager", "--tool-server", "demo"]
289 +
290 + with monkeypatch_class.context() as m:
291 + m.setenv("VLLM_ENABLE_RESPONSES_API_STORE", "1")
292 + m.setenv("PYTHON_EXECUTION_BACKEND", "dangerously_use_uv")
293 + # Helps the model follow instructions better
294 + m.setenv("VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS", "1")
295 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
296 + yield remote_server
297 +
298 + @pytest_asyncio.fixture
299 + async def mcp_disabled_client(self, mcp_disabled_server):
300 + async with mcp_disabled_server.get_async_client() as async_client:
301 + yield async_client
302 +
303 + @pytest.mark.asyncio
304 + @pytest.mark.parametrize("model_name", [MODEL_NAME])
305 + async def test_mcp_tool_env_flag_disabled(
306 + self, mcp_disabled_client: OpenAI, model_name: str
307 + ):
308 + response = await mcp_disabled_client.responses.create(
309 + model=model_name,
310 + input=(
311 + "Execute the following code if the tool is present: "
312 + "import random; print(random.randint(1, 1000000))"
313 + ),
314 + tools=[
315 + {
316 + "type": "mcp",
317 + "server_label": "code_interpreter",
318 + # URL unused for DemoToolServer
319 + "server_url": "http://localhost:8888",
320 + }
321 + ],
322 + extra_body={"enable_response_messages": True},
323 + )
324 + assert response is not None
325 + assert response.status == "completed"
326 + # Verify output messages: No tool calls and responses
327 + tool_call_found = False
328 + tool_response_found = False
329 + for message in response.output_messages:
330 + recipient = message.get("recipient")
331 + if recipient and recipient.startswith("python"):
332 + tool_call_found = True
333 + assert message.get("channel") == "analysis", (
334 + "Tool call should be on analysis channel"
335 + )
336 + author = message.get("author", {})
337 + if (
338 + author.get("role") == "tool"
339 + and author.get("name")
340 + and author.get("name").startswith("python")
341 + ):
342 + tool_response_found = True
343 + assert message.get("channel") == "analysis", (
344 + "Tool response should be on analysis channel"
345 + )
346 +
347 + assert not tool_call_found, "Should not have a python call"
348 + assert not tool_response_found, "Should not have a tool response"
349 + for message in response.input_messages:
350 + assert message.get("author").get("role") != "developer", (
351 + "No developer messages should be present without a valid tool"
352 + )
tests/entrypoints/openai/test_response_api_with_harmony.py
+233 -1
@@ -1,6 +1,5 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import importlib
3 import importlib.util
4 import json
5 import time
@@ -44,6 +43,8 @@ def server():
43 env_dict = dict(
44 VLLM_ENABLE_RESPONSES_API_STORE="1",
45 PYTHON_EXECUTION_BACKEND="dangerously_use_uv",
46 + VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS="code_interpreter,container,web_search_preview",
47 + VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS="1",
48 )
49
50 with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
@@ -855,6 +856,237 @@ async def test_function_calling_with_stream(client: OpenAI, model_name: str):
856 assert event.response.output_text is not None
857
858
859 +@pytest.mark.asyncio
860 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
861 +async def test_function_calling_no_code_interpreter_events(
862 + client: OpenAI, model_name: str
863 +):
864 + """Verify that function calls don't trigger code_interpreter events.
865 +
866 + This test ensures that function calls (functions.*) use their own
867 + function_call event types and don't incorrectly emit code_interpreter
868 + events during streaming.
869 + """
870 + tools = [GET_WEATHER_SCHEMA]
871 + input_list = [
872 + {
873 + "role": "user",
874 + "content": "What's the weather like in Paris today?",
875 + }
876 + ]
877 + stream_response = await client.responses.create(
878 + model=model_name,
879 + input=input_list,
880 + tools=tools,
881 + stream=True,
882 + )
883 +
884 + # Track which event types we see
885 + event_types_seen = set()
886 + function_call_found = False
887 +
888 + async for event in stream_response:
889 + event_types_seen.add(event.type)
890 +
891 + if (
892 + event.type == "response.output_item.added"
893 + and event.item.type == "function_call"
894 + ):
895 + function_call_found = True
896 +
897 + # Ensure NO code_interpreter events are emitted for function calls
898 + assert "code_interpreter" not in event.type, (
899 + "Found code_interpreter event "
900 + f"'{event.type}' during function call. Function calls should only "
901 + "emit function_call events, not code_interpreter events."
902 + )
903 +
904 + # Verify we actually saw a function call
905 + assert function_call_found, "Expected to see a function_call in the stream"
906 +
907 + # Verify we saw the correct function call event types
908 + assert (
909 + "response.function_call_arguments.delta" in event_types_seen
910 + or "response.function_call_arguments.done" in event_types_seen
911 + ), "Expected to see function_call_arguments events"
912 +
913 +
914 +@pytest.mark.asyncio
915 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
916 +async def test_mcp_code_interpreter_streaming(client: OpenAI, model_name: str, server):
917 + tools = [
918 + {
919 + "type": "mcp",
920 + "server_label": "code_interpreter",
921 + }
922 + ]
923 + input_text = (
924 + "Calculate 15 * 32 using python. "
925 + "The python interpreter is not stateful and you must print to see the output."
926 + )
927 +
928 + stream_response = await client.responses.create(
929 + model=model_name,
930 + input=input_text,
931 + tools=tools,
932 + stream=True,
933 + temperature=0.0,
934 + instructions=(
935 + "You must use the Python tool to execute code. Never simulate execution."
936 + ),
937 + )
938 +
939 + mcp_call_added = False
940 + mcp_call_in_progress = False
941 + mcp_arguments_delta_seen = False
942 + mcp_arguments_done = False
943 + mcp_call_completed = False
944 + mcp_item_done = False
945 +
946 + code_interpreter_events_seen = False
947 +
948 + async for event in stream_response:
949 + if "code_interpreter" in event.type:
950 + code_interpreter_events_seen = True
951 +
952 + if event.type == "response.output_item.added":
953 + if hasattr(event.item, "type") and event.item.type == "mcp_call":
954 + mcp_call_added = True
955 + assert event.item.name == "python"
956 + assert event.item.server_label == "code_interpreter"
957 +
958 + elif event.type == "response.mcp_call.in_progress":
959 + mcp_call_in_progress = True
960 +
961 + elif event.type == "response.mcp_call_arguments.delta":
962 + mcp_arguments_delta_seen = True
963 + assert event.delta is not None
964 +
965 + elif event.type == "response.mcp_call_arguments.done":
966 + mcp_arguments_done = True
967 + assert event.name == "python"
968 + assert event.arguments is not None
969 +
970 + elif event.type == "response.mcp_call.completed":
971 + mcp_call_completed = True
972 +
973 + elif (
974 + event.type == "response.output_item.done"
975 + and hasattr(event.item, "type")
976 + and event.item.type == "mcp_call"
977 + ):
978 + mcp_item_done = True
979 + assert event.item.name == "python"
980 + assert event.item.status == "completed"
981 +
982 + assert mcp_call_added, "MCP call was not added"
983 + assert mcp_call_in_progress, "MCP call in_progress event not seen"
984 + assert mcp_arguments_delta_seen, "MCP arguments delta event not seen"
985 + assert mcp_arguments_done, "MCP arguments done event not seen"
986 + assert mcp_call_completed, "MCP call completed event not seen"
987 + assert mcp_item_done, "MCP item done event not seen"
988 +
989 + assert not code_interpreter_events_seen, (
990 + "Should not see code_interpreter events when using MCP type"
991 + )
992 +
993 +
994 +@pytest.mark.asyncio
995 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
996 +async def test_mcp_tool_multi_turn(client: OpenAI, model_name: str, server):
997 + """Test MCP tool calling across multiple turns.
998 +
999 + This test verifies that MCP tools work correctly in multi-turn conversations,
1000 + maintaining state across turns via the previous_response_id mechanism.
1001 + """
1002 + tools = [
1003 + {
1004 + "type": "mcp",
1005 + "server_label": "code_interpreter",
1006 + }
1007 + ]
1008 +
1009 + # First turn - make a calculation
1010 + response1 = await client.responses.create(
1011 + model=model_name,
1012 + input="Calculate 123 * 456 using python and print the result.",
1013 + tools=tools,
1014 + temperature=0.0,
1015 + instructions=(
1016 + "You must use the Python tool to execute code. Never simulate execution."
1017 + ),
1018 + extra_body={"enable_response_messages": True},
1019 + )
1020 +
1021 + assert response1 is not None
1022 + assert response1.status == "completed"
1023 +
1024 + # Verify MCP call in first response by checking output_messages
1025 + tool_call_found = False
1026 + tool_response_found = False
1027 + for message in response1.output_messages:
1028 + recipient = message.get("recipient")
1029 + if recipient and recipient.startswith("python"):
1030 + tool_call_found = True
1031 +
1032 + author = message.get("author", {})
1033 + if (
1034 + author.get("role") == "tool"
1035 + and author.get("name")
1036 + and author.get("name").startswith("python")
1037 + ):
1038 + tool_response_found = True
1039 +
1040 + # Verify MCP tools were actually used
1041 + assert tool_call_found, "MCP tool call not found in output_messages"
1042 + assert tool_response_found, "MCP tool response not found in output_messages"
1043 +
1044 + # Verify input messages: Should have system message with tool, NO developer message
1045 + developer_messages = [
1046 + msg for msg in response1.input_messages if msg["author"]["role"] == "developer"
1047 + ]
1048 + assert len(developer_messages) == 0, (
1049 + "No developer message expected for elevated tools"
1050 + )
1051 +
1052 + # Second turn - reference previous calculation
1053 + response2 = await client.responses.create(
1054 + model=model_name,
1055 + input="Now divide that result by 2.",
1056 + tools=tools,
1057 + temperature=0.0,
1058 + instructions=(
1059 + "You must use the Python tool to execute code. Never simulate execution."
1060 + ),
1061 + previous_response_id=response1.id,
1062 + extra_body={"enable_response_messages": True},
1063 + )
1064 +
1065 + assert response2 is not None
1066 + assert response2.status == "completed"
1067 +
1068 + # Verify input messages are correct: should have two messages -
1069 + # one to the python recipient on analysis channel and one from tool role
1070 + mcp_recipient_messages = []
1071 + tool_role_messages = []
1072 + for msg in response2.input_messages:
1073 + if msg["author"]["role"] == "assistant":
1074 + # Check if this is a message to MCP recipient on analysis channel
1075 + if msg.get("channel") == "analysis" and msg.get("recipient"):
1076 + recipient = msg.get("recipient")
1077 + if recipient.startswith("code_interpreter") or recipient == "python":
1078 + mcp_recipient_messages.append(msg)
1079 + elif msg["author"]["role"] == "tool":
1080 + tool_role_messages.append(msg)
1081 +
1082 + assert len(mcp_recipient_messages) > 0, (
1083 + "Expected message(s) to MCP recipient on analysis channel"
1084 + )
1085 + assert len(tool_role_messages) > 0, (
1086 + "Expected message(s) from tool role after MCP call"
1087 + )
1088 +
1089 +
1090 @pytest.mark.asyncio
1091 @pytest.mark.parametrize("model_name", [MODEL_NAME])
1092 async def test_output_messages_enabled(client: OpenAI, model_name: str, server):
vllm/entrypoints/openai/serving_responses.py
+869 -434
@@ -9,6 +9,7 @@ from collections import deque
9 from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
10 from contextlib import AsyncExitStack
11 from copy import copy
12 +from dataclasses import dataclass
13 from http import HTTPStatus
14 from typing import Final
15
@@ -27,6 +28,10 @@ from openai.types.responses import (
28 ResponseFunctionCallArgumentsDoneEvent,
29 ResponseFunctionToolCall,
30 ResponseFunctionWebSearch,
31 + ResponseMcpCallArgumentsDeltaEvent,
32 + ResponseMcpCallArgumentsDoneEvent,
33 + ResponseMcpCallCompletedEvent,
34 + ResponseMcpCallInProgressEvent,
35 ResponseOutputItem,
36 ResponseOutputItemAddedEvent,
37 ResponseOutputItemDoneEvent,
@@ -44,6 +49,7 @@ from openai.types.responses import (
49 response_function_web_search,
50 response_text_delta_event,
51 )
52 +from openai.types.responses.response_output_item import McpCall
53 from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
54 from openai.types.responses.response_reasoning_item import (
55 Content as ResponseReasoningTextContent,
@@ -119,6 +125,23 @@ from vllm.utils import random_uuid
125 logger = init_logger(__name__)
126
127
128 +@dataclass
129 +class HarmonyStreamingState:
130 + """Mutable state for harmony streaming event processing."""
131 +
132 + current_content_index: int = -1
133 + current_output_index: int = 0
134 + current_item_id: str = ""
135 + sent_output_item_added: bool = False
136 + is_first_function_call_delta: bool = False
137 +
138 + def reset_for_new_item(self) -> None:
139 + """Reset state when expecting a new output item."""
140 + self.current_output_index += 1
141 + self.sent_output_item_added = False
142 + self.is_first_function_call_delta = False
143 +
144 +
145 def _extract_allowed_tools_from_mcp_requests(
146 tools: list[Tool],
147 ) -> dict[str, list[str] | None]:
@@ -740,6 +763,26 @@ class OpenAIServingResponses(OpenAIServing):
763 self.response_store[response.id] = response
764 return response
765
766 + def _is_mcp_tool_by_namespace(self, recipient: str | None) -> bool:
767 + """
768 + Determine if a tool call is an MCP tool based on recipient prefix.
769 +
770 + - Tools starting with "functions." are function calls
771 + - Everything else is an MCP tool
772 + """
773 + if recipient is None:
774 + return False
775 +
776 + # Function calls have "functions." prefix
777 + # Everything else is an MCP tool
778 + return not recipient.startswith("functions.")
779 +
780 + _TOOL_NAME_TO_MCP_SERVER_LABEL: Final[dict[str, str]] = {
781 + "python": "code_interpreter",
782 + "container": "container",
783 + "browser": "web_search_preview",
784 + }
785 +
786 def _topk_logprobs(
787 self,
788 logprobs: dict[int, SampleLogprob],
@@ -1036,8 +1079,7 @@ class OpenAIServingResponses(OpenAIServing):
1079 del prev_msgs[prev_final_msg_idx + 1 :]
1080 for msg in recent_turn_msgs:
1081 assert isinstance(msg, OpenAIHarmonyMessage)
1039 - if msg.channel != "analysis":
1040 - prev_msgs.append(msg)
1082 + prev_msgs.append(msg)
1083 messages.extend(prev_msgs)
1084 # Append the new input.
1085 # Responses API supports simple text inputs without chat format.
@@ -1520,6 +1562,816 @@ class OpenAIServingResponses(OpenAIServing):
1562 )
1563 )
1564
1565 + def _emit_function_call_done_events(
1566 + self,
1567 + previous_item,
1568 + state: HarmonyStreamingState,
1569 + ) -> list[StreamingResponsesResponse]:
1570 + """Emit events when a function call completes."""
1571 + function_name = previous_item.recipient[len("functions.") :]
1572 + events = []
1573 + events.append(
1574 + ResponseFunctionCallArgumentsDoneEvent(
1575 + type="response.function_call_arguments.done",
1576 + arguments=previous_item.content[0].text,
1577 + name=function_name,
1578 + item_id=state.current_item_id,
1579 + output_index=state.current_output_index,
1580 + sequence_number=-1,
1581 + )
1582 + )
1583 + function_call_item = ResponseFunctionToolCall(
1584 + type="function_call",
1585 + arguments=previous_item.content[0].text,
1586 + name=function_name,
1587 + item_id=state.current_item_id,
1588 + output_index=state.current_output_index,
1589 + sequence_number=-1,
1590 + call_id=f"fc_{random_uuid()}",
1591 + status="completed",
1592 + )
1593 + events.append(
1594 + ResponseOutputItemDoneEvent(
1595 + type="response.output_item.done",
1596 + sequence_number=-1,
1597 + output_index=state.current_output_index,
1598 + item=function_call_item,
1599 + )
1600 + )
1601 + return events
1602 +
1603 + def _emit_mcp_call_done_events(
1604 + self,
1605 + previous_item,
1606 + state: HarmonyStreamingState,
1607 + ) -> list[StreamingResponsesResponse]:
1608 + """Emit events when an MCP tool call completes."""
1609 + server_label = self._TOOL_NAME_TO_MCP_SERVER_LABEL.get(
1610 + previous_item.recipient, previous_item.recipient
1611 + )
1612 + events = []
1613 + events.append(
1614 + ResponseMcpCallArgumentsDoneEvent(
1615 + type="response.mcp_call_arguments.done",
1616 + arguments=previous_item.content[0].text,
1617 + name=previous_item.recipient,
1618 + item_id=state.current_item_id,
1619 + output_index=state.current_output_index,
1620 + sequence_number=-1,
1621 + )
1622 + )
1623 + events.append(
1624 + ResponseMcpCallCompletedEvent(
1625 + type="response.mcp_call.completed",
1626 + sequence_number=-1,
1627 + output_index=state.current_output_index,
1628 + item_id=state.current_item_id,
1629 + )
1630 + )
1631 + events.append(
1632 + ResponseOutputItemDoneEvent(
1633 + type="response.output_item.done",
1634 + sequence_number=-1,
1635 + output_index=state.current_output_index,
1636 + item=McpCall(
1637 + type="mcp_call",
1638 + arguments=previous_item.content[0].text,
1639 + name=previous_item.recipient,
1640 + id=state.current_item_id,
1641 + server_label=server_label,
1642 + status="completed",
1643 + ),
1644 + )
1645 + )
1646 + return events
1647 +
1648 + def _emit_reasoning_done_events(
1649 + self,
1650 + previous_item,
1651 + state: HarmonyStreamingState,
1652 + ) -> list[StreamingResponsesResponse]:
1653 + """Emit events when a reasoning (analysis) item completes."""
1654 + content = ResponseReasoningTextContent(
1655 + text=previous_item.content[0].text,
1656 + type="reasoning_text",
1657 + )
1658 + reasoning_item = ResponseReasoningItem(
1659 + type="reasoning",
1660 + content=[content],
1661 + status="completed",
1662 + id=state.current_item_id,
1663 + summary=[],
1664 + )
1665 + events = []
1666 + events.append(
1667 + ResponseReasoningTextDoneEvent(
1668 + type="response.reasoning_text.done",
1669 + item_id=state.current_item_id,
1670 + sequence_number=-1,
1671 + output_index=state.current_output_index,
1672 + content_index=state.current_content_index,
1673 + text=previous_item.content[0].text,
1674 + )
1675 + )
1676 + events.append(
1677 + ResponseReasoningPartDoneEvent(
1678 + type="response.reasoning_part.done",
1679 + sequence_number=-1,
1680 + item_id=state.current_item_id,
1681 + output_index=state.current_output_index,
1682 + content_index=state.current_content_index,
1683 + part=content,
1684 + )
1685 + )
1686 + events.append(
1687 + ResponseOutputItemDoneEvent(
1688 + type="response.output_item.done",
1689 + sequence_number=-1,
1690 + output_index=state.current_output_index,
1691 + item=reasoning_item,
1692 + )
1693 + )
1694 + return events
1695 +
1696 + def _emit_text_output_done_events(
1697 + self,
1698 + previous_item,
1699 + state: HarmonyStreamingState,
1700 + ) -> list[StreamingResponsesResponse]:
1701 + """Emit events when a final text output item completes."""
1702 + text_content = ResponseOutputText(
1703 + type="output_text",
1704 + text=previous_item.content[0].text,
1705 + annotations=[],
1706 + )
1707 + events = []
1708 + events.append(
1709 + ResponseTextDoneEvent(
1710 + type="response.output_text.done",
1711 + sequence_number=-1,
1712 + output_index=state.current_output_index,
1713 + content_index=state.current_content_index,
1714 + text=previous_item.content[0].text,
1715 + logprobs=[],
1716 + item_id=state.current_item_id,
1717 + )
1718 + )
1719 + events.append(
1720 + ResponseContentPartDoneEvent(
1721 + type="response.content_part.done",
1722 + sequence_number=-1,
1723 + item_id=state.current_item_id,
1724 + output_index=state.current_output_index,
1725 + content_index=state.current_content_index,
1726 + part=text_content,
1727 + )
1728 + )
1729 + events.append(
1730 + ResponseOutputItemDoneEvent(
1731 + type="response.output_item.done",
1732 + sequence_number=-1,
1733 + output_index=state.current_output_index,
1734 + item=ResponseOutputMessage(
1735 + id=state.current_item_id,
1736 + type="message",
1737 + role="assistant",
1738 + content=[text_content],
1739 + status="completed",
1740 + ),
1741 + )
1742 + )
1743 + return events
1744 +
1745 + def _emit_previous_item_done_events(
1746 + self,
1747 + previous_item,
1748 + state: HarmonyStreamingState,
1749 + ) -> list[StreamingResponsesResponse]:
1750 + """Emit done events for the previous item when expecting a new start."""
1751 + if previous_item.recipient is not None:
1752 + # Deal with tool call
1753 + if previous_item.recipient.startswith("functions."):
1754 + return self._emit_function_call_done_events(previous_item, state)
1755 + elif (
1756 + self._is_mcp_tool_by_namespace(previous_item.recipient)
1757 + and state.current_item_id is not None
1758 + and state.current_item_id.startswith("mcp_")
1759 + ):
1760 + return self._emit_mcp_call_done_events(previous_item, state)
1761 + elif previous_item.channel == "analysis":
1762 + return self._emit_reasoning_done_events(previous_item, state)
1763 + elif previous_item.channel == "final":
1764 + return self._emit_text_output_done_events(previous_item, state)
1765 + return []
1766 +
1767 + def _emit_final_channel_delta_events(
1768 + self,
1769 + ctx: StreamingHarmonyContext,
1770 + state: HarmonyStreamingState,
1771 + ) -> list[StreamingResponsesResponse]:
1772 + """Emit events for final channel text delta streaming."""
1773 + events = []
1774 + if not state.sent_output_item_added:
1775 + state.sent_output_item_added = True
1776 + state.current_item_id = f"msg_{random_uuid()}"
1777 + events.append(
1778 + ResponseOutputItemAddedEvent(
1779 + type="response.output_item.added",
1780 + sequence_number=-1,
1781 + output_index=state.current_output_index,
1782 + item=ResponseOutputMessage(
1783 + id=state.current_item_id,
1784 + type="message",
1785 + role="assistant",
1786 + content=[],
1787 + status="in_progress",
1788 + ),
1789 + )
1790 + )
1791 + state.current_content_index += 1
1792 + events.append(
1793 + ResponseContentPartAddedEvent(
1794 + type="response.content_part.added",
1795 + sequence_number=-1,
1796 + output_index=state.current_output_index,
1797 + item_id=state.current_item_id,
1798 + content_index=state.current_content_index,
1799 + part=ResponseOutputText(
1800 + type="output_text",
1801 + text="",
1802 + annotations=[],
1803 + logprobs=[],
1804 + ),
1805 + )
1806 + )
1807 + events.append(
1808 + ResponseTextDeltaEvent(
1809 + type="response.output_text.delta",
1810 + sequence_number=-1,
1811 + content_index=state.current_content_index,
1812 + output_index=state.current_output_index,
1813 + item_id=state.current_item_id,
1814 + delta=ctx.parser.last_content_delta,
1815 + # TODO, use logprobs from ctx.last_request_output
1816 + logprobs=[],
1817 + )
1818 + )
1819 + return events
1820 +
1821 + def _emit_analysis_channel_delta_events(
1822 + self,
1823 + ctx: StreamingHarmonyContext,
1824 + state: HarmonyStreamingState,
1825 + ) -> list[StreamingResponsesResponse]:
1826 + """Emit events for analysis channel reasoning delta streaming."""
1827 + events = []
1828 + if not state.sent_output_item_added:
1829 + state.sent_output_item_added = True
1830 + state.current_item_id = f"msg_{random_uuid()}"
1831 + events.append(
1832 + ResponseOutputItemAddedEvent(
1833 + type="response.output_item.added",
1834 + sequence_number=-1,
1835 + output_index=state.current_output_index,
1836 + item=ResponseReasoningItem(
1837 + type="reasoning",
1838 + id=state.current_item_id,
1839 + summary=[],
1840 + status="in_progress",
1841 + ),
1842 + )
1843 + )
1844 + state.current_content_index += 1
1845 + events.append(
1846 + ResponseReasoningPartAddedEvent(
1847 + type="response.reasoning_part.added",
1848 + sequence_number=-1,
1849 + output_index=state.current_output_index,
1850 + item_id=state.current_item_id,
1851 + content_index=state.current_content_index,
1852 + part=ResponseReasoningTextContent(
1853 + text="",
1854 + type="reasoning_text",
1855 + ),
1856 + )
1857 + )
1858 + events.append(
1859 + ResponseReasoningTextDeltaEvent(
1860 + type="response.reasoning_text.delta",
1861 + item_id=state.current_item_id,
1862 + output_index=state.current_output_index,
1863 + content_index=state.current_content_index,
1864 + delta=ctx.parser.last_content_delta,
1865 + sequence_number=-1,
1866 + )
1867 + )
1868 + return events
1869 +
1870 + def _emit_mcp_tool_delta_events(
1871 + self,
1872 + ctx: StreamingHarmonyContext,
1873 + state: HarmonyStreamingState,
1874 + recipient: str,
1875 + ) -> list[StreamingResponsesResponse]:
1876 + """Emit events for MCP tool delta streaming."""
1877 + server_label = self._TOOL_NAME_TO_MCP_SERVER_LABEL.get(recipient, recipient)
1878 + events = []
1879 + if not state.sent_output_item_added:
1880 + state.sent_output_item_added = True
1881 + state.current_item_id = f"mcp_{random_uuid()}"
1882 + events.append(
1883 + ResponseOutputItemAddedEvent(
1884 + type="response.output_item.added",
1885 + sequence_number=-1,
1886 + output_index=state.current_output_index,
1887 + item=McpCall(
1888 + type="mcp_call",
1889 + id=state.current_item_id,
1890 + name=recipient,
1891 + arguments="",
1892 + server_label=server_label,
1893 + status="in_progress",
1894 + ),
1895 + )
1896 + )
1897 + events.append(
1898 + ResponseMcpCallInProgressEvent(
1899 + type="response.mcp_call.in_progress",
1900 + sequence_number=-1,
1901 + output_index=state.current_output_index,
1902 + item_id=state.current_item_id,
1903 + )
1904 + )
1905 + events.append(
1906 + ResponseMcpCallArgumentsDeltaEvent(
1907 + type="response.mcp_call_arguments.delta",
1908 + sequence_number=-1,
1909 + output_index=state.current_output_index,
1910 + item_id=state.current_item_id,
1911 + delta=ctx.parser.last_content_delta,
1912 + )
1913 + )
1914 + return events
1915 +
1916 + def _emit_code_interpreter_delta_events(
1917 + self,
1918 + ctx: StreamingHarmonyContext,
1919 + state: HarmonyStreamingState,
1920 + ) -> list[StreamingResponsesResponse]:
1921 + """Emit events for code interpreter delta streaming."""
1922 + events = []
1923 + if not state.sent_output_item_added:
1924 + state.sent_output_item_added = True
1925 + state.current_item_id = f"tool_{random_uuid()}"
1926 + events.append(
1927 + ResponseOutputItemAddedEvent(
1928 + type="response.output_item.added",
1929 + sequence_number=-1,
1930 + output_index=state.current_output_index,
1931 + item=ResponseCodeInterpreterToolCallParam(
1932 + type="code_interpreter_call",
1933 + id=state.current_item_id,
1934 + code=None,
1935 + container_id="auto",
1936 + outputs=None,
1937 + status="in_progress",
1938 + ),
1939 + )
1940 + )
1941 + events.append(
1942 + ResponseCodeInterpreterCallInProgressEvent(
1943 + type="response.code_interpreter_call.in_progress",
1944 + sequence_number=-1,
1945 + output_index=state.current_output_index,
1946 + item_id=state.current_item_id,
1947 + )
1948 + )
1949 + events.append(
1950 + ResponseCodeInterpreterCallCodeDeltaEvent(
1951 + type="response.code_interpreter_call_code.delta",
1952 + sequence_number=-1,
1953 + output_index=state.current_output_index,
1954 + item_id=state.current_item_id,
1955 + delta=ctx.parser.last_content_delta,
1956 + )
1957 + )
1958 + return events
1959 +
1960 + def _emit_mcp_prefix_delta_events(
1961 + self,
1962 + ctx: StreamingHarmonyContext,
1963 + state: HarmonyStreamingState,
1964 + ) -> list[StreamingResponsesResponse]:
1965 + """Emit events for MCP prefix (mcp.*) delta streaming."""
1966 + events = []
1967 + if not state.sent_output_item_added:
1968 + state.sent_output_item_added = True
1969 + state.current_item_id = f"mcp_{random_uuid()}"
1970 + mcp_name = ctx.parser.current_recipient[len("mcp.") :]
1971 +
1972 + events.append(
1973 + ResponseOutputItemAddedEvent(
1974 + type="response.output_item.added",
1975 + sequence_number=-1,
1976 + output_index=state.current_output_index,
1977 + item=McpCall(
1978 + type="mcp_call",
1979 + id=state.current_item_id,
1980 + name=mcp_name,
1981 + arguments="",
1982 + server_label=mcp_name,
1983 + status="in_progress",
1984 + ),
1985 + )
1986 + )
1987 + events.append(
1988 + ResponseMcpCallInProgressEvent(
1989 + type="response.mcp_call.in_progress",
1990 + sequence_number=-1,
1991 + output_index=state.current_output_index,
1992 + item_id=state.current_item_id,
1993 + )
1994 + )
1995 +
1996 + events.append(
1997 + ResponseMcpCallArgumentsDeltaEvent(
1998 + type="response.mcp_call_arguments.delta",
1999 + sequence_number=-1,
2000 + output_index=state.current_output_index,
2001 + item_id=state.current_item_id,
2002 + delta=ctx.parser.last_content_delta,
2003 + )
2004 + )
2005 + return events
2006 +
2007 + def _emit_content_delta_events(
2008 + self,
2009 + ctx: StreamingHarmonyContext,
2010 + state: HarmonyStreamingState,
2011 + ) -> list[StreamingResponsesResponse]:
2012 + """Emit events for content delta streaming based on channel type."""
2013 + if not ctx.parser.last_content_delta:
2014 + return []
2015 +
2016 + if (
2017 + ctx.parser.current_channel == "final"
2018 + and ctx.parser.current_recipient is None
2019 + ):
2020 + return self._emit_final_channel_delta_events(ctx, state)
2021 + elif (
2022 + ctx.parser.current_channel == "analysis"
2023 + and ctx.parser.current_recipient is None
2024 + ):
2025 + return self._emit_analysis_channel_delta_events(ctx, state)
2026 + # built-in tools will be triggered on the analysis channel
2027 + # However, occasionally built-in tools will
2028 + # still be output to commentary.
2029 + elif (
2030 + ctx.parser.current_channel == "commentary"
2031 + or ctx.parser.current_channel == "analysis"
2032 + ) and ctx.parser.current_recipient is not None:
2033 + recipient = ctx.parser.current_recipient
2034 + # Check for function calls first - they have their own event handling
2035 + if recipient.startswith("functions."):
2036 + return self._emit_function_call_delta_events(ctx, state)
2037 + is_mcp_tool = self._is_mcp_tool_by_namespace(recipient)
2038 + if is_mcp_tool:
2039 + return self._emit_mcp_tool_delta_events(ctx, state, recipient)
2040 + else:
2041 + return self._emit_code_interpreter_delta_events(ctx, state)
2042 + elif (
2043 + (
2044 + ctx.parser.current_channel == "commentary"
2045 + or ctx.parser.current_channel == "analysis"
2046 + )
2047 + and ctx.parser.current_recipient is not None
2048 + and ctx.parser.current_recipient.startswith("mcp.")
2049 + ):
2050 + return self._emit_mcp_prefix_delta_events(ctx, state)
2051 +
2052 + return []
2053 +
2054 + def _emit_browser_tool_events(
2055 + self,
2056 + previous_item,
2057 + state: HarmonyStreamingState,
2058 + ) -> list[StreamingResponsesResponse]:
2059 + """Emit events for browser tool calls (web search)."""
2060 + function_name = previous_item.recipient[len("browser.") :]
2061 + parsed_args = json.loads(previous_item.content[0].text)
2062 + action = None
2063 +
2064 + if function_name == "search":
2065 + action = response_function_web_search.ActionSearch(
2066 + type="search",
2067 + query=parsed_args["query"],
2068 + )
2069 + elif function_name == "open":
2070 + action = response_function_web_search.ActionOpenPage(
2071 + type="open_page",
2072 + # TODO: translate to url
2073 + url=f"cursor:{parsed_args.get('cursor', '')}",
2074 + )
2075 + elif function_name == "find":
2076 + action = response_function_web_search.ActionFind(
2077 + type="find",
2078 + pattern=parsed_args["pattern"],
2079 + # TODO: translate to url
2080 + url=f"cursor:{parsed_args.get('cursor', '')}",
2081 + )
2082 + else:
2083 + raise ValueError(f"Unknown function name: {function_name}")
2084 +
2085 + state.current_item_id = f"tool_{random_uuid()}"
2086 + events = []
2087 + events.append(
2088 + ResponseOutputItemAddedEvent(
2089 + type="response.output_item.added",
2090 + sequence_number=-1,
2091 + output_index=state.current_output_index,
2092 + item=response_function_web_search.ResponseFunctionWebSearch(
2093 + # TODO: generate a unique id for web search call
2094 + type="web_search_call",
2095 + id=state.current_item_id,
2096 + action=action,
2097 + status="in_progress",
2098 + ),
2099 + )
2100 + )
2101 + events.append(
2102 + ResponseWebSearchCallInProgressEvent(
2103 + type="response.web_search_call.in_progress",
2104 + sequence_number=-1,
2105 + output_index=state.current_output_index,
2106 + item_id=state.current_item_id,
2107 + )
2108 + )
2109 + events.append(
2110 + ResponseWebSearchCallSearchingEvent(
2111 + type="response.web_search_call.searching",
2112 + sequence_number=-1,
2113 + output_index=state.current_output_index,
2114 + item_id=state.current_item_id,
2115 + )
2116 + )
2117 + # enqueue
2118 + events.append(
2119 + ResponseWebSearchCallCompletedEvent(
2120 + type="response.web_search_call.completed",
2121 + sequence_number=-1,
2122 + output_index=state.current_output_index,
2123 + item_id=state.current_item_id,
2124 + )
2125 + )
2126 + events.append(
2127 + ResponseOutputItemDoneEvent(
2128 + type="response.output_item.done",
2129 + sequence_number=-1,
2130 + output_index=state.current_output_index,
2131 + item=ResponseFunctionWebSearch(
2132 + type="web_search_call",
2133 + id=state.current_item_id,
2134 + action=action,
2135 + status="completed",
2136 + ),
2137 + )
2138 + )
2139 + return events
2140 +
2141 + def _emit_mcp_tool_completion_events(
2142 + self,
2143 + previous_item,
2144 + state: HarmonyStreamingState,
2145 + ) -> list[StreamingResponsesResponse]:
2146 + """Emit events when an MCP tool completes during assistant action turn."""
2147 + recipient = previous_item.recipient
2148 + server_label = self._TOOL_NAME_TO_MCP_SERVER_LABEL.get(recipient, recipient)
2149 + events = []
2150 + events.append(
2151 + ResponseMcpCallArgumentsDoneEvent(
2152 + type="response.mcp_call_arguments.done",
2153 + sequence_number=-1,
2154 + output_index=state.current_output_index,
2155 + item_id=state.current_item_id,
2156 + arguments=previous_item.content[0].text,
2157 + name=recipient,
2158 + )
2159 + )
2160 + events.append(
2161 + ResponseMcpCallCompletedEvent(
2162 + type="response.mcp_call.completed",
2163 + sequence_number=-1,
2164 + output_index=state.current_output_index,
2165 + item_id=state.current_item_id,
2166 + )
2167 + )
2168 + events.append(
2169 + ResponseOutputItemDoneEvent(
2170 + type="response.output_item.done",
2171 + sequence_number=-1,
2172 + output_index=state.current_output_index,
2173 + item=McpCall(
2174 + type="mcp_call",
2175 + id=state.current_item_id,
2176 + name=recipient,
2177 + arguments=previous_item.content[0].text,
2178 + server_label=server_label,
2179 + status="completed",
2180 + ),
2181 + )
2182 + )
2183 + return events
2184 +
2185 + def _emit_code_interpreter_completion_events(
2186 + self,
2187 + previous_item,
2188 + state: HarmonyStreamingState,
2189 + ) -> list[StreamingResponsesResponse]:
2190 + """Emit events when code interpreter completes."""
2191 + events = []
2192 + events.append(
2193 + ResponseCodeInterpreterCallCodeDoneEvent(
2194 + type="response.code_interpreter_call_code.done",
2195 + sequence_number=-1,
2196 + output_index=state.current_output_index,
2197 + item_id=state.current_item_id,
2198 + code=previous_item.content[0].text,
2199 + )
2200 + )
2201 + events.append(
2202 + ResponseCodeInterpreterCallInterpretingEvent(
2203 + type="response.code_interpreter_call.interpreting",
2204 + sequence_number=-1,
2205 + output_index=state.current_output_index,
2206 + item_id=state.current_item_id,
2207 + )
2208 + )
2209 + events.append(
2210 + ResponseCodeInterpreterCallCompletedEvent(
2211 + type="response.code_interpreter_call.completed",
2212 + sequence_number=-1,
2213 + output_index=state.current_output_index,
2214 + item_id=state.current_item_id,
2215 + )
2216 + )
2217 + events.append(
2218 + ResponseOutputItemDoneEvent(
2219 + type="response.output_item.done",
2220 + sequence_number=-1,
2221 + output_index=state.current_output_index,
2222 + item=ResponseCodeInterpreterToolCallParam(
2223 + type="code_interpreter_call",
2224 + id=state.current_item_id,
2225 + code=previous_item.content[0].text,
2226 + container_id="auto",
2227 + outputs=[],
2228 + status="completed",
2229 + ),
2230 + )
2231 + )
2232 + return events
2233 +
2234 + def _emit_mcp_prefix_completion_events(
2235 + self,
2236 + previous_item,
2237 + state: HarmonyStreamingState,
2238 + ) -> list[StreamingResponsesResponse]:
2239 + """Emit events when an MCP prefix tool (mcp.*) completes."""
2240 + mcp_name = previous_item.recipient[len("mcp.") :]
2241 + events = []
2242 + events.append(
2243 + ResponseMcpCallArgumentsDoneEvent(
2244 + type="response.mcp_call_arguments.done",
2245 + sequence_number=-1,
2246 + output_index=state.current_output_index,
2247 + item_id=state.current_item_id,
2248 + arguments=previous_item.content[0].text,
2249 + name=mcp_name,
2250 + )
2251 + )
2252 + events.append(
2253 + ResponseMcpCallCompletedEvent(
2254 + type="response.mcp_call.completed",
2255 + sequence_number=-1,
2256 + output_index=state.current_output_index,
2257 + item_id=state.current_item_id,
2258 + )
2259 + )
2260 + events.append(
2261 + ResponseOutputItemDoneEvent(
2262 + type="response.output_item.done",
2263 + sequence_number=-1,
2264 + output_index=state.current_output_index,
2265 + item=McpCall(
2266 + type="mcp_call",
2267 + id=state.current_item_id,
2268 + name=mcp_name,
2269 + arguments=previous_item.content[0].text,
2270 + server_label=mcp_name,
2271 + status="completed",
2272 + ),
2273 + )
2274 + )
2275 + return events
2276 +
2277 + def _emit_tool_action_events(
2278 + self,
2279 + ctx: StreamingHarmonyContext,
2280 + state: HarmonyStreamingState,
2281 + ) -> list[StreamingResponsesResponse]:
2282 + """Emit events for tool action turn."""
2283 + if not ctx.is_assistant_action_turn() or len(ctx.parser.messages) == 0:
2284 + return []
2285 +
2286 + events = []
2287 + previous_item = ctx.parser.messages[-1]
2288 +
2289 + # Handle browser tool
2290 + if (
2291 + self.tool_server is not None
2292 + and self.tool_server.has_tool("browser")
2293 + and previous_item.recipient is not None
2294 + and previous_item.recipient.startswith("browser.")
2295 + ):
2296 + events.extend(self._emit_browser_tool_events(previous_item, state))
2297 +
2298 + # Handle tool completion
2299 + if (
2300 + self.tool_server is not None
2301 + and previous_item.recipient is not None
2302 + and state.current_item_id is not None
2303 + and state.sent_output_item_added
2304 + ):
2305 + recipient = previous_item.recipient
2306 + # Handle MCP prefix tool completion first
2307 + if recipient.startswith("mcp."):
2308 + events.extend(
2309 + self._emit_mcp_prefix_completion_events(previous_item, state)
2310 + )
2311 + else:
2312 + # Handle other MCP tool and code interpreter completion
2313 + is_mcp_tool = self._is_mcp_tool_by_namespace(
2314 + recipient
2315 + ) and state.current_item_id.startswith("mcp_")
2316 + if is_mcp_tool:
2317 + events.extend(
2318 + self._emit_mcp_tool_completion_events(previous_item, state)
2319 + )
2320 + else:
2321 + events.extend(
2322 + self._emit_code_interpreter_completion_events(
2323 + previous_item, state
2324 + )
2325 + )
2326 +
2327 + return events
2328 +
2329 + def _emit_function_call_delta_events(
2330 + self,
2331 + ctx: StreamingHarmonyContext,
2332 + state: HarmonyStreamingState,
2333 + ) -> list[StreamingResponsesResponse]:
2334 + """Emit events for developer function calls on commentary channel."""
2335 + if not (
2336 + ctx.parser.current_channel == "commentary"
2337 + and ctx.parser.current_recipient
2338 + and ctx.parser.current_recipient.startswith("functions.")
2339 + ):
2340 + return []
2341 +
2342 + events = []
2343 + if state.is_first_function_call_delta is False:
2344 + state.is_first_function_call_delta = True
2345 + fc_name = ctx.parser.current_recipient[len("functions.") :]
2346 + state.current_item_id = f"fc_{random_uuid()}"
2347 + tool_call_item = ResponseFunctionToolCall(
2348 + name=fc_name,
2349 + type="function_call",
2350 + id=state.current_item_id,
2351 + call_id=f"call_{random_uuid()}",
2352 + arguments="",
2353 + status="in_progress",
2354 + )
2355 + events.append(
2356 + ResponseOutputItemAddedEvent(
2357 + type="response.output_item.added",
2358 + sequence_number=-1,
2359 + output_index=state.current_output_index,
2360 + item=tool_call_item,
2361 + )
2362 + )
2363 + # Always emit the delta (including on first call)
2364 + events.append(
2365 + ResponseFunctionCallArgumentsDeltaEvent(
2366 + item_id=state.current_item_id,
2367 + delta=ctx.parser.last_content_delta,
2368 + output_index=state.current_output_index,
2369 + sequence_number=-1,
2370 + type="response.function_call_arguments.delta",
2371 + )
2372 + )
2373 + return events
2374 +
2375 async def _process_harmony_streaming_events(
2376 self,
2377 request: ResponsesRequest,
@@ -1534,11 +2386,8 @@ class OpenAIServingResponses(OpenAIServing):
2386 [StreamingResponsesResponse], StreamingResponsesResponse
2387 ],
2388 ) -> AsyncGenerator[StreamingResponsesResponse, None]:
1537 - current_content_index = -1
1538 - current_output_index = 0
1539 - current_item_id: str = ""
1540 - sent_output_item_added = False
1541 - is_first_function_call_delta = False
2389 + state = HarmonyStreamingState()
2390 +
2391 async for ctx in result_generator:
2392 assert isinstance(ctx, StreamingHarmonyContext)
2393
@@ -1546,435 +2395,21 @@ class OpenAIServingResponses(OpenAIServing):
2395 self._raise_if_error(ctx.finish_reason, request.request_id)
2396
2397 if ctx.is_expecting_start():
1549 - current_output_index += 1
1550 - sent_output_item_added = False
1551 - is_first_function_call_delta = False
2398 if len(ctx.parser.messages) > 0:
2399 previous_item = ctx.parser.messages[-1]
1554 - if previous_item.recipient is not None:
1555 - # Deal with tool call
1556 - if previous_item.recipient.startswith("functions."):
1557 - function_name = previous_item.recipient[len("functions.") :]
1558 - yield _increment_sequence_number_and_return(
1559 - ResponseFunctionCallArgumentsDoneEvent(
1560 - type="response.function_call_arguments.done",
1561 - arguments=previous_item.content[0].text,
1562 - name=function_name,
1563 - item_id=current_item_id,
1564 - output_index=current_output_index,
1565 - sequence_number=-1,
1566 - )
1567 - )
1568 - function_call_item = ResponseFunctionToolCall(
1569 - type="function_call",
1570 - arguments=previous_item.content[0].text,
1571 - name=function_name,
1572 - item_id=current_item_id,
1573 - output_index=current_output_index,
1574 - sequence_number=-1,
1575 - call_id=f"fc_{random_uuid()}",
1576 - status="completed",
1577 - )
1578 - yield _increment_sequence_number_and_return(
1579 - ResponseOutputItemDoneEvent(
1580 - type="response.output_item.done",
1581 - sequence_number=-1,
1582 - output_index=current_output_index,
1583 - item=function_call_item,
1584 - )
1585 - )
1586 - elif previous_item.channel == "analysis":
1587 - content = ResponseReasoningTextContent(
1588 - text=previous_item.content[0].text,
1589 - type="reasoning_text",
1590 - )
1591 - reasoning_item = ResponseReasoningItem(
1592 - type="reasoning",
1593 - content=[content],
1594 - status="completed",
1595 - id=current_item_id,
1596 - summary=[],
1597 - )
1598 - yield _increment_sequence_number_and_return(
1599 - ResponseReasoningTextDoneEvent(
1600 - type="response.reasoning_text.done",
1601 - item_id=current_item_id,
1602 - sequence_number=-1,
1603 - output_index=current_output_index,
1604 - content_index=current_content_index,
1605 - text=previous_item.content[0].text,
1606 - )
1607 - )
1608 - yield _increment_sequence_number_and_return(
1609 - ResponseReasoningPartDoneEvent(
1610 - type="response.reasoning_part.done",
1611 - sequence_number=-1,
1612 - item_id=current_item_id,
1613 - output_index=current_output_index,
1614 - content_index=current_content_index,
1615 - part=content,
1616 - )
1617 - )
1618 - yield _increment_sequence_number_and_return(
1619 - ResponseOutputItemDoneEvent(
1620 - type="response.output_item.done",
1621 - sequence_number=-1,
1622 - output_index=current_output_index,
1623 - item=reasoning_item,
1624 - )
1625 - )
1626 - elif previous_item.channel == "final":
1627 - text_content = ResponseOutputText(
1628 - type="output_text",
1629 - text=previous_item.content[0].text,
1630 - annotations=[],
1631 - )
1632 - yield _increment_sequence_number_and_return(
1633 - ResponseTextDoneEvent(
1634 - type="response.output_text.done",
1635 - sequence_number=-1,
1636 - output_index=current_output_index,
1637 - content_index=current_content_index,
1638 - text=previous_item.content[0].text,
1639 - logprobs=[],
1640 - item_id=current_item_id,
1641 - )
1642 - )
1643 - yield _increment_sequence_number_and_return(
1644 - ResponseContentPartDoneEvent(
1645 - type="response.content_part.done",
1646 - sequence_number=-1,
1647 - item_id=current_item_id,
1648 - output_index=current_output_index,
1649 - content_index=current_content_index,
1650 - part=text_content,
1651 - )
1652 - )
1653 - yield _increment_sequence_number_and_return(
1654 - ResponseOutputItemDoneEvent(
1655 - type="response.output_item.done",
1656 - sequence_number=-1,
1657 - output_index=current_output_index,
1658 - item=ResponseOutputMessage(
1659 - id=current_item_id,
1660 - type="message",
1661 - role="assistant",
1662 - content=[text_content],
1663 - status="completed",
1664 - ),
1665 - )
1666 - )
1667 -
1668 - # stream the output of a harmony message
1669 - if ctx.parser.last_content_delta:
1670 - if (
1671 - ctx.parser.current_channel == "final"
1672 - and ctx.parser.current_recipient is None
1673 - ):
1674 - if not sent_output_item_added:
1675 - sent_output_item_added = True
1676 - current_item_id = f"msg_{random_uuid()}"
1677 - yield _increment_sequence_number_and_return(
1678 - ResponseOutputItemAddedEvent(
1679 - type="response.output_item.added",
1680 - sequence_number=-1,
1681 - output_index=current_output_index,
1682 - item=ResponseOutputMessage(
1683 - id=current_item_id,
1684 - type="message",
1685 - role="assistant",
1686 - content=[],
1687 - status="in_progress",
1688 - ),
1689 - )
1690 - )
1691 - current_content_index += 1
1692 - yield _increment_sequence_number_and_return(
1693 - ResponseContentPartAddedEvent(
1694 - type="response.content_part.added",
1695 - sequence_number=-1,
1696 - output_index=current_output_index,
1697 - item_id=current_item_id,
1698 - content_index=current_content_index,
1699 - part=ResponseOutputText(
1700 - type="output_text",
1701 - text="",
1702 - annotations=[],
1703 - logprobs=[],
1704 - ),
1705 - )
1706 - )
1707 - yield _increment_sequence_number_and_return(
1708 - ResponseTextDeltaEvent(
1709 - type="response.output_text.delta",
1710 - sequence_number=-1,
1711 - content_index=current_content_index,
1712 - output_index=current_output_index,
1713 - item_id=current_item_id,
1714 - delta=ctx.parser.last_content_delta,
1715 - # TODO, use logprobs from ctx.last_request_output
1716 - logprobs=[],
1717 - )
1718 - )
1719 - elif (
1720 - ctx.parser.current_channel == "analysis"
1721 - and ctx.parser.current_recipient is None
1722 - ):
1723 - if not sent_output_item_added:
1724 - sent_output_item_added = True
1725 - current_item_id = f"msg_{random_uuid()}"
1726 - yield _increment_sequence_number_and_return(
1727 - ResponseOutputItemAddedEvent(
1728 - type="response.output_item.added",
1729 - sequence_number=-1,
1730 - output_index=current_output_index,
1731 - item=ResponseReasoningItem(
1732 - type="reasoning",
1733 - id=current_item_id,
1734 - summary=[],
1735 - status="in_progress",
1736 - ),
1737 - )
1738 - )
1739 - current_content_index += 1
1740 - yield _increment_sequence_number_and_return(
1741 - ResponseReasoningPartAddedEvent(
1742 - type="response.reasoning_part.added",
1743 - sequence_number=-1,
1744 - output_index=current_output_index,
1745 - item_id=current_item_id,
1746 - content_index=current_content_index,
1747 - part=ResponseReasoningTextContent(
1748 - text="",
1749 - type="reasoning_text",
1750 - ),
1751 - )
1752 - )
1753 - yield _increment_sequence_number_and_return(
1754 - ResponseReasoningTextDeltaEvent(
1755 - type="response.reasoning_text.delta",
1756 - item_id=current_item_id,
1757 - output_index=current_output_index,
1758 - content_index=current_content_index,
1759 - delta=ctx.parser.last_content_delta,
1760 - sequence_number=-1,
1761 - )
1762 - )
1763 - # built-in tools will be triggered on the analysis channel
1764 - # However, occasionally built-in tools will
1765 - # still be output to commentary.
1766 - elif (
1767 - ctx.parser.current_channel == "commentary"
1768 - or ctx.parser.current_channel == "analysis"
1769 - ) and ctx.parser.current_recipient == "python":
1770 - if not sent_output_item_added:
1771 - sent_output_item_added = True
1772 - current_item_id = f"tool_{random_uuid()}"
1773 - yield _increment_sequence_number_and_return(
1774 - ResponseOutputItemAddedEvent(
1775 - type="response.output_item.added",
1776 - sequence_number=-1,
1777 - output_index=current_output_index,
1778 - item=ResponseCodeInterpreterToolCallParam(
1779 - type="code_interpreter_call",
1780 - id=current_item_id,
1781 - code=None,
1782 - container_id="auto",
1783 - outputs=None,
1784 - status="in_progress",
1785 - ),
1786 - )
1787 - )
1788 - yield _increment_sequence_number_and_return(
1789 - ResponseCodeInterpreterCallInProgressEvent(
1790 - type="response.code_interpreter_call.in_progress",
1791 - sequence_number=-1,
1792 - output_index=current_output_index,
1793 - item_id=current_item_id,
1794 - )
1795 - )
1796 - yield _increment_sequence_number_and_return(
1797 - ResponseCodeInterpreterCallCodeDeltaEvent(
1798 - type="response.code_interpreter_call_code.delta",
1799 - sequence_number=-1,
1800 - output_index=current_output_index,
1801 - item_id=current_item_id,
1802 - delta=ctx.parser.last_content_delta,
1803 - )
1804 - )
1805 -
1806 - # stream tool call outputs
1807 - if ctx.is_assistant_action_turn() and len(ctx.parser.messages) > 0:
1808 - previous_item = ctx.parser.messages[-1]
1809 - if (
1810 - self.tool_server is not None
1811 - and self.tool_server.has_tool("browser")
1812 - and previous_item.recipient is not None
1813 - and previous_item.recipient.startswith("browser.")
1814 - ):
1815 - function_name = previous_item.recipient[len("browser.") :]
1816 - action = None
1817 - parsed_args = json.loads(previous_item.content[0].text)
1818 - if function_name == "search":
1819 - action = response_function_web_search.ActionSearch(
1820 - type="search",
1821 - query=parsed_args["query"],
1822 - )
1823 - elif function_name == "open":
1824 - action = response_function_web_search.ActionOpenPage(
1825 - type="open_page",
1826 - # TODO: translate to url
1827 - url=f"cursor:{parsed_args.get('cursor', '')}",
1828 - )
1829 - elif function_name == "find":
1830 - action = response_function_web_search.ActionFind(
1831 - type="find",
1832 - pattern=parsed_args["pattern"],
1833 - # TODO: translate to url
1834 - url=f"cursor:{parsed_args.get('cursor', '')}",
1835 - )
1836 - else:
1837 - raise ValueError(f"Unknown function name: {function_name}")
1838 -
1839 - current_item_id = f"tool_{random_uuid()}"
1840 - yield _increment_sequence_number_and_return(
1841 - ResponseOutputItemAddedEvent(
1842 - type="response.output_item.added",
1843 - sequence_number=-1,
1844 - output_index=current_output_index,
1845 - item=response_function_web_search.ResponseFunctionWebSearch(
1846 - # TODO: generate a unique id for web search call
1847 - type="web_search_call",
1848 - id=current_item_id,
1849 - action=action,
1850 - status="in_progress",
1851 - ),
1852 - )
1853 - )
1854 - yield _increment_sequence_number_and_return(
1855 - ResponseWebSearchCallInProgressEvent(
1856 - type="response.web_search_call.in_progress",
1857 - sequence_number=-1,
1858 - output_index=current_output_index,
1859 - item_id=current_item_id,
1860 - )
1861 - )
1862 - yield _increment_sequence_number_and_return(
1863 - ResponseWebSearchCallSearchingEvent(
1864 - type="response.web_search_call.searching",
1865 - sequence_number=-1,
1866 - output_index=current_output_index,
1867 - item_id=current_item_id,
1868 - )
1869 - )
1870 -
1871 - # enqueue
1872 - yield _increment_sequence_number_and_return(
1873 - ResponseWebSearchCallCompletedEvent(
1874 - type="response.web_search_call.completed",
1875 - sequence_number=-1,
1876 - output_index=current_output_index,
1877 - item_id=current_item_id,
1878 - )
1879 - )
1880 - yield _increment_sequence_number_and_return(
1881 - ResponseOutputItemDoneEvent(
1882 - type="response.output_item.done",
1883 - sequence_number=-1,
1884 - output_index=current_output_index,
1885 - item=ResponseFunctionWebSearch(
1886 - type="web_search_call",
1887 - id=current_item_id,
1888 - action=action,
1889 - status="completed",
1890 - ),
1891 - )
1892 - )
1893 -
1894 - if (
1895 - self.tool_server is not None
1896 - and self.tool_server.has_tool("python")
1897 - and previous_item.recipient is not None
1898 - and previous_item.recipient.startswith("python")
1899 - ):
1900 - yield _increment_sequence_number_and_return(
1901 - ResponseCodeInterpreterCallCodeDoneEvent(
1902 - type="response.code_interpreter_call_code.done",
1903 - sequence_number=-1,
1904 - output_index=current_output_index,
1905 - item_id=current_item_id,
1906 - code=previous_item.content[0].text,
1907 - )
1908 - )
1909 - yield _increment_sequence_number_and_return(
1910 - ResponseCodeInterpreterCallInterpretingEvent(
1911 - type="response.code_interpreter_call.interpreting",
1912 - sequence_number=-1,
1913 - output_index=current_output_index,
1914 - item_id=current_item_id,
1915 - )
1916 - )
1917 - yield _increment_sequence_number_and_return(
1918 - ResponseCodeInterpreterCallCompletedEvent(
1919 - type="response.code_interpreter_call.completed",
1920 - sequence_number=-1,
1921 - output_index=current_output_index,
1922 - item_id=current_item_id,
1923 - )
1924 - )
1925 - yield _increment_sequence_number_and_return(
1926 - ResponseOutputItemDoneEvent(
1927 - type="response.output_item.done",
1928 - sequence_number=-1,
1929 - output_index=current_output_index,
1930 - item=ResponseCodeInterpreterToolCallParam(
1931 - type="code_interpreter_call",
1932 - id=current_item_id,
1933 - code=previous_item.content[0].text,
1934 - container_id="auto",
1935 - # TODO: add outputs here
1936 - outputs=[],
1937 - status="completed",
1938 - ),
1939 - )
1940 - )
1941 - # developer tools will be triggered on the commentary channel
1942 - # and recipient starts with "functions.TOOL_NAME"
1943 - if (
1944 - ctx.parser.current_channel == "commentary"
1945 - and ctx.parser.current_recipient
1946 - and ctx.parser.current_recipient.startswith("functions.")
1947 - ):
1948 - if is_first_function_call_delta is False:
1949 - is_first_function_call_delta = True
1950 - fc_name = ctx.parser.current_recipient[len("functions.") :]
1951 - tool_call_item = ResponseFunctionToolCall(
1952 - name=fc_name,
1953 - type="function_call",
1954 - id=current_item_id,
1955 - call_id=f"call_{random_uuid()}",
1956 - arguments="",
1957 - status="in_progress",
1958 - )
1959 - current_item_id = f"fc_{random_uuid()}"
1960 - yield _increment_sequence_number_and_return(
1961 - ResponseOutputItemAddedEvent(
1962 - type="response.output_item.added",
1963 - sequence_number=-1,
1964 - output_index=current_output_index,
1965 - item=tool_call_item,
1966 - )
1967 - )
1968 - else:
1969 - yield _increment_sequence_number_and_return(
1970 - ResponseFunctionCallArgumentsDeltaEvent(
1971 - item_id=current_item_id,
1972 - delta=ctx.parser.last_content_delta,
1973 - output_index=current_output_index,
1974 - sequence_number=-1,
1975 - type="response.function_call_arguments.delta",
1976 - )
1977 - )
2400 + for event in self._emit_previous_item_done_events(
2401 + previous_item, state
2402 + ):
2403 + yield _increment_sequence_number_and_return(event)
2404 + state.reset_for_new_item()
2405 +
2406 + # Stream the output of a harmony message
2407 + for event in self._emit_content_delta_events(ctx, state):
2408 + yield _increment_sequence_number_and_return(event)
2409 +
2410 + # Stream tool call outputs
2411 + for event in self._emit_tool_action_events(ctx, state):
2412 + yield _increment_sequence_number_and_return(event)
2413
2414 async def responses_stream_generator(
2415 self,