@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ed31a020ee
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 01:15 UTC

[Refactor] Extract Harmony streaming SSE event builders into streaming_events.py (#34909)

Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>

Flora Feng committed Feb 20, 2026 at 09:20 UTC ed31a020ee5e383a069a59750261a307bd8ddde4
2 files changed +907 -874
vllm/entrypoints/openai/responses/serving.py
+10 -874
@@ -2,36 +2,22 @@
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import asyncio
5 -import json
5 import time
6 import uuid
7 from collections import deque
8 from collections.abc import AsyncGenerator, AsyncIterator, Callable, Sequence
9 from contextlib import AsyncExitStack
10 from copy import copy
12 -from dataclasses import dataclass, replace
11 +from dataclasses import replace
12 from http import HTTPStatus
13 from typing import Final
14
15 import jinja2
16 from fastapi import Request
17 from openai.types.responses import (
19 - ResponseCodeInterpreterCallCodeDeltaEvent,
20 - ResponseCodeInterpreterCallCodeDoneEvent,
21 - ResponseCodeInterpreterCallCompletedEvent,
22 - ResponseCodeInterpreterCallInProgressEvent,
23 - ResponseCodeInterpreterCallInterpretingEvent,
24 - ResponseCodeInterpreterToolCallParam,
18 ResponseContentPartAddedEvent,
19 ResponseContentPartDoneEvent,
27 - ResponseFunctionCallArgumentsDeltaEvent,
28 - ResponseFunctionCallArgumentsDoneEvent,
20 ResponseFunctionToolCall,
30 - ResponseFunctionWebSearch,
31 - ResponseMcpCallArgumentsDeltaEvent,
32 - ResponseMcpCallArgumentsDoneEvent,
33 - ResponseMcpCallCompletedEvent,
34 - ResponseMcpCallInProgressEvent,
21 ResponseOutputItem,
22 ResponseOutputItemAddedEvent,
23 ResponseOutputItemDoneEvent,
@@ -43,13 +29,8 @@ from openai.types.responses import (
29 ResponseStatus,
30 ResponseTextDeltaEvent,
31 ResponseTextDoneEvent,
46 - ResponseWebSearchCallCompletedEvent,
47 - ResponseWebSearchCallInProgressEvent,
48 - ResponseWebSearchCallSearchingEvent,
49 - response_function_web_search,
32 response_text_delta_event,
33 )
52 -from openai.types.responses.response_output_item import McpCall
34 from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
35 from openai.types.responses.response_reasoning_item import (
36 Content as ResponseReasoningTextContent,
@@ -102,13 +83,17 @@ from vllm.entrypoints.openai.responses.protocol import (
83 ResponseCreatedEvent,
84 ResponseInProgressEvent,
85 ResponseInputOutputMessage,
105 - ResponseReasoningPartAddedEvent,
106 - ResponseReasoningPartDoneEvent,
86 ResponsesRequest,
87 ResponsesResponse,
88 ResponseUsage,
89 StreamingResponsesResponse,
90 )
91 +from vllm.entrypoints.openai.responses.streaming_events import (
92 + HarmonyStreamingState,
93 + emit_content_delta_events,
94 + emit_previous_item_done_events,
95 + emit_tool_action_events,
96 +)
97 from vllm.entrypoints.openai.responses.utils import (
98 construct_input_messages,
99 construct_tool_dicts,
@@ -129,23 +114,6 @@ from vllm.utils import random_uuid
114 logger = init_logger(__name__)
115
116
132 -@dataclass
133 -class HarmonyStreamingState:
134 - """Mutable state for harmony streaming event processing."""
135 -
136 - current_content_index: int = -1
137 - current_output_index: int = 0
138 - current_item_id: str = ""
139 - sent_output_item_added: bool = False
140 - is_first_function_call_delta: bool = False
141 -
142 - def reset_for_new_item(self) -> None:
143 - """Reset state when expecting a new output item."""
144 - self.current_output_index += 1
145 - self.sent_output_item_added = False
146 - self.is_first_function_call_delta = False
147 -
148 -
117 def _extract_allowed_tools_from_mcp_requests(
118 tools: list[Tool],
119 ) -> dict[str, list[str] | None]:
@@ -817,26 +785,6 @@ class OpenAIServingResponses(OpenAIServing):
785 self.response_store[response.id] = response
786 return response
787
820 - def _is_mcp_tool_by_namespace(self, recipient: str | None) -> bool:
821 - """
822 - Determine if a tool call is an MCP tool based on recipient prefix.
823 -
824 - - Tools starting with "functions." are function calls
825 - - Everything else is an MCP tool
826 - """
827 - if recipient is None:
828 - return False
829 -
830 - # Function calls have "functions." prefix
831 - # Everything else is an MCP tool
832 - return not recipient.startswith("functions.")
833 -
834 - _TOOL_NAME_TO_MCP_SERVER_LABEL: Final[dict[str, str]] = {
835 - "python": "code_interpreter",
836 - "container": "container",
837 - "browser": "web_search_preview",
838 - }
839 -
788 def _topk_logprobs(
789 self,
790 logprobs: dict[int, SampleLogprob],
@@ -1605,816 +1553,6 @@ class OpenAIServingResponses(OpenAIServing):
1553 )
1554 )
1555
1608 - def _emit_function_call_done_events(
1609 - self,
1610 - previous_item,
1611 - state: HarmonyStreamingState,
1612 - ) -> list[StreamingResponsesResponse]:
1613 - """Emit events when a function call completes."""
1614 - function_name = previous_item.recipient[len("functions.") :]
1615 - events = []
1616 - events.append(
1617 - ResponseFunctionCallArgumentsDoneEvent(
1618 - type="response.function_call_arguments.done",
1619 - arguments=previous_item.content[0].text,
1620 - name=function_name,
1621 - item_id=state.current_item_id,
1622 - output_index=state.current_output_index,
1623 - sequence_number=-1,
1624 - )
1625 - )
1626 - function_call_item = ResponseFunctionToolCall(
1627 - type="function_call",
1628 - arguments=previous_item.content[0].text,
1629 - name=function_name,
1630 - item_id=state.current_item_id,
1631 - output_index=state.current_output_index,
1632 - sequence_number=-1,
1633 - call_id=f"fc_{random_uuid()}",
1634 - status="completed",
1635 - )
1636 - events.append(
1637 - ResponseOutputItemDoneEvent(
1638 - type="response.output_item.done",
1639 - sequence_number=-1,
1640 - output_index=state.current_output_index,
1641 - item=function_call_item,
1642 - )
1643 - )
1644 - return events
1645 -
1646 - def _emit_mcp_call_done_events(
1647 - self,
1648 - previous_item,
1649 - state: HarmonyStreamingState,
1650 - ) -> list[StreamingResponsesResponse]:
1651 - """Emit events when an MCP tool call completes."""
1652 - server_label = self._TOOL_NAME_TO_MCP_SERVER_LABEL.get(
1653 - previous_item.recipient, previous_item.recipient
1654 - )
1655 - events = []
1656 - events.append(
1657 - ResponseMcpCallArgumentsDoneEvent(
1658 - type="response.mcp_call_arguments.done",
1659 - arguments=previous_item.content[0].text,
1660 - name=previous_item.recipient,
1661 - item_id=state.current_item_id,
1662 - output_index=state.current_output_index,
1663 - sequence_number=-1,
1664 - )
1665 - )
1666 - events.append(
1667 - ResponseMcpCallCompletedEvent(
1668 - type="response.mcp_call.completed",
1669 - sequence_number=-1,
1670 - output_index=state.current_output_index,
1671 - item_id=state.current_item_id,
1672 - )
1673 - )
1674 - events.append(
1675 - ResponseOutputItemDoneEvent(
1676 - type="response.output_item.done",
1677 - sequence_number=-1,
1678 - output_index=state.current_output_index,
1679 - item=McpCall(
1680 - type="mcp_call",
1681 - arguments=previous_item.content[0].text,
1682 - name=previous_item.recipient,
1683 - id=state.current_item_id,
1684 - server_label=server_label,
1685 - status="completed",
1686 - ),
1687 - )
1688 - )
1689 - return events
1690 -
1691 - def _emit_reasoning_done_events(
1692 - self,
1693 - previous_item,
1694 - state: HarmonyStreamingState,
1695 - ) -> list[StreamingResponsesResponse]:
1696 - """Emit events when a reasoning (analysis) item completes."""
1697 - content = ResponseReasoningTextContent(
1698 - text=previous_item.content[0].text,
1699 - type="reasoning_text",
1700 - )
1701 - reasoning_item = ResponseReasoningItem(
1702 - type="reasoning",
1703 - content=[content],
1704 - status="completed",
1705 - id=state.current_item_id,
1706 - summary=[],
1707 - )
1708 - events = []
1709 - events.append(
1710 - ResponseReasoningTextDoneEvent(
1711 - type="response.reasoning_text.done",
1712 - item_id=state.current_item_id,
1713 - sequence_number=-1,
1714 - output_index=state.current_output_index,
1715 - content_index=state.current_content_index,
1716 - text=previous_item.content[0].text,
1717 - )
1718 - )
1719 - events.append(
1720 - ResponseReasoningPartDoneEvent(
1721 - type="response.reasoning_part.done",
1722 - sequence_number=-1,
1723 - item_id=state.current_item_id,
1724 - output_index=state.current_output_index,
1725 - content_index=state.current_content_index,
1726 - part=content,
1727 - )
1728 - )
1729 - events.append(
1730 - ResponseOutputItemDoneEvent(
1731 - type="response.output_item.done",
1732 - sequence_number=-1,
1733 - output_index=state.current_output_index,
1734 - item=reasoning_item,
1735 - )
1736 - )
1737 - return events
1738 -
1739 - def _emit_text_output_done_events(
1740 - self,
1741 - previous_item,
1742 - state: HarmonyStreamingState,
1743 - ) -> list[StreamingResponsesResponse]:
1744 - """Emit events when a final text output item completes."""
1745 - text_content = ResponseOutputText(
1746 - type="output_text",
1747 - text=previous_item.content[0].text,
1748 - annotations=[],
1749 - )
1750 - events = []
1751 - events.append(
1752 - ResponseTextDoneEvent(
1753 - type="response.output_text.done",
1754 - sequence_number=-1,
1755 - output_index=state.current_output_index,
1756 - content_index=state.current_content_index,
1757 - text=previous_item.content[0].text,
1758 - logprobs=[],
1759 - item_id=state.current_item_id,
1760 - )
1761 - )
1762 - events.append(
1763 - ResponseContentPartDoneEvent(
1764 - type="response.content_part.done",
1765 - sequence_number=-1,
1766 - item_id=state.current_item_id,
1767 - output_index=state.current_output_index,
1768 - content_index=state.current_content_index,
1769 - part=text_content,
1770 - )
1771 - )
1772 - events.append(
1773 - ResponseOutputItemDoneEvent(
1774 - type="response.output_item.done",
1775 - sequence_number=-1,
1776 - output_index=state.current_output_index,
1777 - item=ResponseOutputMessage(
1778 - id=state.current_item_id,
1779 - type="message",
1780 - role="assistant",
1781 - content=[text_content],
1782 - status="completed",
1783 - ),
1784 - )
1785 - )
1786 - return events
1787 -
1788 - def _emit_previous_item_done_events(
1789 - self,
1790 - previous_item,
1791 - state: HarmonyStreamingState,
1792 - ) -> list[StreamingResponsesResponse]:
1793 - """Emit done events for the previous item when expecting a new start."""
1794 - if previous_item.recipient is not None:
1795 - # Deal with tool call
1796 - if previous_item.recipient.startswith("functions."):
1797 - return self._emit_function_call_done_events(previous_item, state)
1798 - elif (
1799 - self._is_mcp_tool_by_namespace(previous_item.recipient)
1800 - and state.current_item_id is not None
1801 - and state.current_item_id.startswith("mcp_")
1802 - ):
1803 - return self._emit_mcp_call_done_events(previous_item, state)
1804 - elif previous_item.channel == "analysis":
1805 - return self._emit_reasoning_done_events(previous_item, state)
1806 - elif previous_item.channel == "final":
1807 - return self._emit_text_output_done_events(previous_item, state)
1808 - return []
1809 -
1810 - def _emit_final_channel_delta_events(
1811 - self,
1812 - ctx: StreamingHarmonyContext,
1813 - state: HarmonyStreamingState,
1814 - ) -> list[StreamingResponsesResponse]:
1815 - """Emit events for final channel text delta streaming."""
1816 - events = []
1817 - if not state.sent_output_item_added:
1818 - state.sent_output_item_added = True
1819 - state.current_item_id = f"msg_{random_uuid()}"
1820 - events.append(
1821 - ResponseOutputItemAddedEvent(
1822 - type="response.output_item.added",
1823 - sequence_number=-1,
1824 - output_index=state.current_output_index,
1825 - item=ResponseOutputMessage(
1826 - id=state.current_item_id,
1827 - type="message",
1828 - role="assistant",
1829 - content=[],
1830 - status="in_progress",
1831 - ),
1832 - )
1833 - )
1834 - state.current_content_index += 1
1835 - events.append(
1836 - ResponseContentPartAddedEvent(
1837 - type="response.content_part.added",
1838 - sequence_number=-1,
1839 - output_index=state.current_output_index,
1840 - item_id=state.current_item_id,
1841 - content_index=state.current_content_index,
1842 - part=ResponseOutputText(
1843 - type="output_text",
1844 - text="",
1845 - annotations=[],
1846 - logprobs=[],
1847 - ),
1848 - )
1849 - )
1850 - events.append(
1851 - ResponseTextDeltaEvent(
1852 - type="response.output_text.delta",
1853 - sequence_number=-1,
1854 - content_index=state.current_content_index,
1855 - output_index=state.current_output_index,
1856 - item_id=state.current_item_id,
1857 - delta=ctx.last_content_delta,
1858 - # TODO, use logprobs from ctx.last_request_output
1859 - logprobs=[],
1860 - )
1861 - )
1862 - return events
1863 -
1864 - def _emit_analysis_channel_delta_events(
1865 - self,
1866 - ctx: StreamingHarmonyContext,
1867 - state: HarmonyStreamingState,
1868 - ) -> list[StreamingResponsesResponse]:
1869 - """Emit events for analysis channel reasoning delta streaming."""
1870 - events = []
1871 - if not state.sent_output_item_added:
1872 - state.sent_output_item_added = True
1873 - state.current_item_id = f"msg_{random_uuid()}"
1874 - events.append(
1875 - ResponseOutputItemAddedEvent(
1876 - type="response.output_item.added",
1877 - sequence_number=-1,
1878 - output_index=state.current_output_index,
1879 - item=ResponseReasoningItem(
1880 - type="reasoning",
1881 - id=state.current_item_id,
1882 - summary=[],
1883 - status="in_progress",
1884 - ),
1885 - )
1886 - )
1887 - state.current_content_index += 1
1888 - events.append(
1889 - ResponseReasoningPartAddedEvent(
1890 - type="response.reasoning_part.added",
1891 - sequence_number=-1,
1892 - output_index=state.current_output_index,
1893 - item_id=state.current_item_id,
1894 - content_index=state.current_content_index,
1895 - part=ResponseReasoningTextContent(
1896 - text="",
1897 - type="reasoning_text",
1898 - ),
1899 - )
1900 - )
1901 - events.append(
1902 - ResponseReasoningTextDeltaEvent(
1903 - type="response.reasoning_text.delta",
1904 - item_id=state.current_item_id,
1905 - output_index=state.current_output_index,
1906 - content_index=state.current_content_index,
1907 - delta=ctx.last_content_delta,
1908 - sequence_number=-1,
1909 - )
1910 - )
1911 - return events
1912 -
1913 - def _emit_mcp_tool_delta_events(
1914 - self,
1915 - ctx: StreamingHarmonyContext,
1916 - state: HarmonyStreamingState,
1917 - recipient: str,
1918 - ) -> list[StreamingResponsesResponse]:
1919 - """Emit events for MCP tool delta streaming."""
1920 - server_label = self._TOOL_NAME_TO_MCP_SERVER_LABEL.get(recipient, recipient)
1921 - events = []
1922 - if not state.sent_output_item_added:
1923 - state.sent_output_item_added = True
1924 - state.current_item_id = f"mcp_{random_uuid()}"
1925 - events.append(
1926 - ResponseOutputItemAddedEvent(
1927 - type="response.output_item.added",
1928 - sequence_number=-1,
1929 - output_index=state.current_output_index,
1930 - item=McpCall(
1931 - type="mcp_call",
1932 - id=state.current_item_id,
1933 - name=recipient,
1934 - arguments="",
1935 - server_label=server_label,
1936 - status="in_progress",
1937 - ),
1938 - )
1939 - )
1940 - events.append(
1941 - ResponseMcpCallInProgressEvent(
1942 - type="response.mcp_call.in_progress",
1943 - sequence_number=-1,
1944 - output_index=state.current_output_index,
1945 - item_id=state.current_item_id,
1946 - )
1947 - )
1948 - events.append(
1949 - ResponseMcpCallArgumentsDeltaEvent(
1950 - type="response.mcp_call_arguments.delta",
1951 - sequence_number=-1,
1952 - output_index=state.current_output_index,
1953 - item_id=state.current_item_id,
1954 - delta=ctx.last_content_delta,
1955 - )
1956 - )
1957 - return events
1958 -
1959 - def _emit_code_interpreter_delta_events(
1960 - self,
1961 - ctx: StreamingHarmonyContext,
1962 - state: HarmonyStreamingState,
1963 - ) -> list[StreamingResponsesResponse]:
1964 - """Emit events for code interpreter delta streaming."""
1965 - events = []
1966 - if not state.sent_output_item_added:
1967 - state.sent_output_item_added = True
1968 - state.current_item_id = f"tool_{random_uuid()}"
1969 - events.append(
1970 - ResponseOutputItemAddedEvent(
1971 - type="response.output_item.added",
1972 - sequence_number=-1,
1973 - output_index=state.current_output_index,
1974 - item=ResponseCodeInterpreterToolCallParam(
1975 - type="code_interpreter_call",
1976 - id=state.current_item_id,
1977 - code=None,
1978 - container_id="auto",
1979 - outputs=None,
1980 - status="in_progress",
1981 - ),
1982 - )
1983 - )
1984 - events.append(
1985 - ResponseCodeInterpreterCallInProgressEvent(
1986 - type="response.code_interpreter_call.in_progress",
1987 - sequence_number=-1,
1988 - output_index=state.current_output_index,
1989 - item_id=state.current_item_id,
1990 - )
1991 - )
1992 - events.append(
1993 - ResponseCodeInterpreterCallCodeDeltaEvent(
1994 - type="response.code_interpreter_call_code.delta",
1995 - sequence_number=-1,
1996 - output_index=state.current_output_index,
1997 - item_id=state.current_item_id,
1998 - delta=ctx.last_content_delta,
1999 - )
2000 - )
2001 - return events
2002 -
2003 - def _emit_mcp_prefix_delta_events(
2004 - self,
2005 - ctx: StreamingHarmonyContext,
2006 - state: HarmonyStreamingState,
2007 - ) -> list[StreamingResponsesResponse]:
2008 - """Emit events for MCP prefix (mcp.*) delta streaming."""
2009 - events = []
2010 - if not state.sent_output_item_added:
2011 - state.sent_output_item_added = True
2012 - state.current_item_id = f"mcp_{random_uuid()}"
2013 - mcp_name = ctx.parser.current_recipient[len("mcp.") :]
2014 -
2015 - events.append(
2016 - ResponseOutputItemAddedEvent(
2017 - type="response.output_item.added",
2018 - sequence_number=-1,
2019 - output_index=state.current_output_index,
2020 - item=McpCall(
2021 - type="mcp_call",
2022 - id=state.current_item_id,
2023 - name=mcp_name,
2024 - arguments="",
2025 - server_label=mcp_name,
2026 - status="in_progress",
2027 - ),
2028 - )
2029 - )
2030 - events.append(
2031 - ResponseMcpCallInProgressEvent(
2032 - type="response.mcp_call.in_progress",
2033 - sequence_number=-1,
2034 - output_index=state.current_output_index,
2035 - item_id=state.current_item_id,
2036 - )
2037 - )
2038 -
2039 - events.append(
2040 - ResponseMcpCallArgumentsDeltaEvent(
2041 - type="response.mcp_call_arguments.delta",
2042 - sequence_number=-1,
2043 - output_index=state.current_output_index,
2044 - item_id=state.current_item_id,
2045 - delta=ctx.last_content_delta,
2046 - )
2047 - )
2048 - return events
2049 -
2050 - def _emit_content_delta_events(
2051 - self,
2052 - ctx: StreamingHarmonyContext,
2053 - state: HarmonyStreamingState,
2054 - ) -> list[StreamingResponsesResponse]:
2055 - """Emit events for content delta streaming based on channel type."""
2056 - if not ctx.last_content_delta:
2057 - return []
2058 -
2059 - if (
2060 - ctx.parser.current_channel == "final"
2061 - and ctx.parser.current_recipient is None
2062 - ):
2063 - return self._emit_final_channel_delta_events(ctx, state)
2064 - elif (
2065 - ctx.parser.current_channel == "analysis"
2066 - and ctx.parser.current_recipient is None
2067 - ):
2068 - return self._emit_analysis_channel_delta_events(ctx, state)
2069 - # built-in tools will be triggered on the analysis channel
2070 - # However, occasionally built-in tools will
2071 - # still be output to commentary.
2072 - elif (
2073 - ctx.parser.current_channel == "commentary"
2074 - or ctx.parser.current_channel == "analysis"
2075 - ) and ctx.parser.current_recipient is not None:
2076 - recipient = ctx.parser.current_recipient
2077 - # Check for function calls first - they have their own event handling
2078 - if recipient.startswith("functions."):
2079 - return self._emit_function_call_delta_events(ctx, state)
2080 - is_mcp_tool = self._is_mcp_tool_by_namespace(recipient)
2081 - if is_mcp_tool:
2082 - return self._emit_mcp_tool_delta_events(ctx, state, recipient)
2083 - else:
2084 - return self._emit_code_interpreter_delta_events(ctx, state)
2085 - elif (
2086 - (
2087 - ctx.parser.current_channel == "commentary"
2088 - or ctx.parser.current_channel == "analysis"
2089 - )
2090 - and ctx.parser.current_recipient is not None
2091 - and ctx.parser.current_recipient.startswith("mcp.")
2092 - ):
2093 - return self._emit_mcp_prefix_delta_events(ctx, state)
2094 -
2095 - return []
2096 -
2097 - def _emit_browser_tool_events(
2098 - self,
2099 - previous_item,
2100 - state: HarmonyStreamingState,
2101 - ) -> list[StreamingResponsesResponse]:
2102 - """Emit events for browser tool calls (web search)."""
2103 - function_name = previous_item.recipient[len("browser.") :]
2104 - parsed_args = json.loads(previous_item.content[0].text)
2105 - action = None
2106 -
2107 - if function_name == "search":
2108 - action = response_function_web_search.ActionSearch(
2109 - type="search",
2110 - query=parsed_args["query"],
2111 - )
2112 - elif function_name == "open":
2113 - action = response_function_web_search.ActionOpenPage(
2114 - type="open_page",
2115 - # TODO: translate to url
2116 - url=f"cursor:{parsed_args.get('cursor', '')}",
2117 - )
2118 - elif function_name == "find":
2119 - action = response_function_web_search.ActionFind(
2120 - type="find",
2121 - pattern=parsed_args["pattern"],
2122 - # TODO: translate to url
2123 - url=f"cursor:{parsed_args.get('cursor', '')}",
2124 - )
2125 - else:
2126 - raise ValueError(f"Unknown function name: {function_name}")
2127 -
2128 - state.current_item_id = f"tool_{random_uuid()}"
2129 - events = []
2130 - events.append(
2131 - ResponseOutputItemAddedEvent(
2132 - type="response.output_item.added",
2133 - sequence_number=-1,
2134 - output_index=state.current_output_index,
2135 - item=response_function_web_search.ResponseFunctionWebSearch(
2136 - # TODO: generate a unique id for web search call
2137 - type="web_search_call",
2138 - id=state.current_item_id,
2139 - action=action,
2140 - status="in_progress",
2141 - ),
2142 - )
2143 - )
2144 - events.append(
2145 - ResponseWebSearchCallInProgressEvent(
2146 - type="response.web_search_call.in_progress",
2147 - sequence_number=-1,
2148 - output_index=state.current_output_index,
2149 - item_id=state.current_item_id,
2150 - )
2151 - )
2152 - events.append(
2153 - ResponseWebSearchCallSearchingEvent(
2154 - type="response.web_search_call.searching",
2155 - sequence_number=-1,
2156 - output_index=state.current_output_index,
2157 - item_id=state.current_item_id,
2158 - )
2159 - )
2160 - # enqueue
2161 - events.append(
2162 - ResponseWebSearchCallCompletedEvent(
2163 - type="response.web_search_call.completed",
2164 - sequence_number=-1,
2165 - output_index=state.current_output_index,
2166 - item_id=state.current_item_id,
2167 - )
2168 - )
2169 - events.append(
2170 - ResponseOutputItemDoneEvent(
2171 - type="response.output_item.done",
2172 - sequence_number=-1,
2173 - output_index=state.current_output_index,
2174 - item=ResponseFunctionWebSearch(
2175 - type="web_search_call",
2176 - id=state.current_item_id,
2177 - action=action,
2178 - status="completed",
2179 - ),
2180 - )
2181 - )
2182 - return events
2183 -
2184 - def _emit_mcp_tool_completion_events(
2185 - self,
2186 - previous_item,
2187 - state: HarmonyStreamingState,
2188 - ) -> list[StreamingResponsesResponse]:
2189 - """Emit events when an MCP tool completes during assistant action turn."""
2190 - recipient = previous_item.recipient
2191 - server_label = self._TOOL_NAME_TO_MCP_SERVER_LABEL.get(recipient, recipient)
2192 - events = []
2193 - events.append(
2194 - ResponseMcpCallArgumentsDoneEvent(
2195 - type="response.mcp_call_arguments.done",
2196 - sequence_number=-1,
2197 - output_index=state.current_output_index,
2198 - item_id=state.current_item_id,
2199 - arguments=previous_item.content[0].text,
2200 - name=recipient,
2201 - )
2202 - )
2203 - events.append(
2204 - ResponseMcpCallCompletedEvent(
2205 - type="response.mcp_call.completed",
2206 - sequence_number=-1,
2207 - output_index=state.current_output_index,
2208 - item_id=state.current_item_id,
2209 - )
2210 - )
2211 - events.append(
2212 - ResponseOutputItemDoneEvent(
2213 - type="response.output_item.done",
2214 - sequence_number=-1,
2215 - output_index=state.current_output_index,
2216 - item=McpCall(
2217 - type="mcp_call",
2218 - id=state.current_item_id,
2219 - name=recipient,
2220 - arguments=previous_item.content[0].text,
2221 - server_label=server_label,
2222 - status="completed",
2223 - ),
2224 - )
2225 - )
2226 - return events
2227 -
2228 - def _emit_code_interpreter_completion_events(
2229 - self,
2230 - previous_item,
2231 - state: HarmonyStreamingState,
2232 - ) -> list[StreamingResponsesResponse]:
2233 - """Emit events when code interpreter completes."""
2234 - events = []
2235 - events.append(
2236 - ResponseCodeInterpreterCallCodeDoneEvent(
2237 - type="response.code_interpreter_call_code.done",
2238 - sequence_number=-1,
2239 - output_index=state.current_output_index,
2240 - item_id=state.current_item_id,
2241 - code=previous_item.content[0].text,
2242 - )
2243 - )
2244 - events.append(
2245 - ResponseCodeInterpreterCallInterpretingEvent(
2246 - type="response.code_interpreter_call.interpreting",
2247 - sequence_number=-1,
2248 - output_index=state.current_output_index,
2249 - item_id=state.current_item_id,
2250 - )
2251 - )
2252 - events.append(
2253 - ResponseCodeInterpreterCallCompletedEvent(
2254 - type="response.code_interpreter_call.completed",
2255 - sequence_number=-1,
2256 - output_index=state.current_output_index,
2257 - item_id=state.current_item_id,
2258 - )
2259 - )
2260 - events.append(
2261 - ResponseOutputItemDoneEvent(
2262 - type="response.output_item.done",
2263 - sequence_number=-1,
2264 - output_index=state.current_output_index,
2265 - item=ResponseCodeInterpreterToolCallParam(
2266 - type="code_interpreter_call",
2267 - id=state.current_item_id,
2268 - code=previous_item.content[0].text,
2269 - container_id="auto",
2270 - outputs=[],
2271 - status="completed",
2272 - ),
2273 - )
2274 - )
2275 - return events
2276 -
2277 - def _emit_mcp_prefix_completion_events(
2278 - self,
2279 - previous_item,
2280 - state: HarmonyStreamingState,
2281 - ) -> list[StreamingResponsesResponse]:
2282 - """Emit events when an MCP prefix tool (mcp.*) completes."""
2283 - mcp_name = previous_item.recipient[len("mcp.") :]
2284 - events = []
2285 - events.append(
2286 - ResponseMcpCallArgumentsDoneEvent(
2287 - type="response.mcp_call_arguments.done",
2288 - sequence_number=-1,
2289 - output_index=state.current_output_index,
2290 - item_id=state.current_item_id,
2291 - arguments=previous_item.content[0].text,
2292 - name=mcp_name,
2293 - )
2294 - )
2295 - events.append(
2296 - ResponseMcpCallCompletedEvent(
2297 - type="response.mcp_call.completed",
2298 - sequence_number=-1,
2299 - output_index=state.current_output_index,
2300 - item_id=state.current_item_id,
2301 - )
2302 - )
2303 - events.append(
2304 - ResponseOutputItemDoneEvent(
2305 - type="response.output_item.done",
2306 - sequence_number=-1,
2307 - output_index=state.current_output_index,
2308 - item=McpCall(
2309 - type="mcp_call",
2310 - id=state.current_item_id,
2311 - name=mcp_name,
2312 - arguments=previous_item.content[0].text,
2313 - server_label=mcp_name,
2314 - status="completed",
2315 - ),
2316 - )
2317 - )
2318 - return events
2319 -
2320 - def _emit_tool_action_events(
2321 - self,
2322 - ctx: StreamingHarmonyContext,
2323 - state: HarmonyStreamingState,
2324 - ) -> list[StreamingResponsesResponse]:
2325 - """Emit events for tool action turn."""
2326 - if not ctx.is_assistant_action_turn() or len(ctx.parser.messages) == 0:
2327 - return []
2328 -
2329 - events = []
2330 - previous_item = ctx.parser.messages[-1]
2331 -
2332 - # Handle browser tool
2333 - if (
2334 - self.tool_server is not None
2335 - and self.tool_server.has_tool("browser")
2336 - and previous_item.recipient is not None
2337 - and previous_item.recipient.startswith("browser.")
2338 - ):
2339 - events.extend(self._emit_browser_tool_events(previous_item, state))
2340 -
2341 - # Handle tool completion
2342 - if (
2343 - self.tool_server is not None
2344 - and previous_item.recipient is not None
2345 - and state.current_item_id is not None
2346 - and state.sent_output_item_added
2347 - ):
2348 - recipient = previous_item.recipient
2349 - # Handle MCP prefix tool completion first
2350 - if recipient.startswith("mcp."):
2351 - events.extend(
2352 - self._emit_mcp_prefix_completion_events(previous_item, state)
2353 - )
2354 - else:
2355 - # Handle other MCP tool and code interpreter completion
2356 - is_mcp_tool = self._is_mcp_tool_by_namespace(
2357 - recipient
2358 - ) and state.current_item_id.startswith("mcp_")
2359 - if is_mcp_tool:
2360 - events.extend(
2361 - self._emit_mcp_tool_completion_events(previous_item, state)
2362 - )
2363 - else:
2364 - events.extend(
2365 - self._emit_code_interpreter_completion_events(
2366 - previous_item, state
2367 - )
2368 - )
2369 -
2370 - return events
2371 -
2372 - def _emit_function_call_delta_events(
2373 - self,
2374 - ctx: StreamingHarmonyContext,
2375 - state: HarmonyStreamingState,
2376 - ) -> list[StreamingResponsesResponse]:
2377 - """Emit events for developer function calls on commentary channel."""
2378 - if not (
2379 - ctx.parser.current_channel == "commentary"
2380 - and ctx.parser.current_recipient
2381 - and ctx.parser.current_recipient.startswith("functions.")
2382 - ):
2383 - return []
2384 -
2385 - events = []
2386 - if state.is_first_function_call_delta is False:
2387 - state.is_first_function_call_delta = True
2388 - fc_name = ctx.parser.current_recipient[len("functions.") :]
2389 - state.current_item_id = f"fc_{random_uuid()}"
2390 - tool_call_item = ResponseFunctionToolCall(
2391 - name=fc_name,
2392 - type="function_call",
2393 - id=state.current_item_id,
2394 - call_id=f"call_{random_uuid()}",
2395 - arguments="",
2396 - status="in_progress",
2397 - )
2398 - events.append(
2399 - ResponseOutputItemAddedEvent(
2400 - type="response.output_item.added",
2401 - sequence_number=-1,
2402 - output_index=state.current_output_index,
2403 - item=tool_call_item,
2404 - )
2405 - )
2406 - # Always emit the delta (including on first call)
2407 - events.append(
2408 - ResponseFunctionCallArgumentsDeltaEvent(
2409 - item_id=state.current_item_id,
2410 - delta=ctx.last_content_delta,
2411 - output_index=state.current_output_index,
2412 - sequence_number=-1,
2413 - type="response.function_call_arguments.delta",
2414 - )
2415 - )
2416 - return events
2417 -
1556 async def _process_harmony_streaming_events(
1557 self,
1558 request: ResponsesRequest,
@@ -2440,18 +1578,16 @@ class OpenAIServingResponses(OpenAIServing):
1578 if ctx.is_expecting_start():
1579 if len(ctx.parser.messages) > 0:
1580 previous_item = ctx.parser.messages[-1]
2443 - for event in self._emit_previous_item_done_events(
2444 - previous_item, state
2445 - ):
1581 + for event in emit_previous_item_done_events(previous_item, state):
1582 yield _increment_sequence_number_and_return(event)
1583 state.reset_for_new_item()
1584
1585 # Stream the output of a harmony message
2450 - for event in self._emit_content_delta_events(ctx, state):
1586 + for event in emit_content_delta_events(ctx, state):
1587 yield _increment_sequence_number_and_return(event)
1588
1589 # Stream tool call outputs
2454 - for event in self._emit_tool_action_events(ctx, state):
1590 + for event in emit_tool_action_events(ctx, state, self.tool_server):
1591 yield _increment_sequence_number_and_return(event)
1592
1593 async def responses_stream_generator(
vllm/entrypoints/openai/responses/streaming_events.py new
+897
@@ -0,0 +1,897 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +Streaming SSE event builders for the Responses API.
5 +
6 +Pure functions that translate streaming state + delta data into
7 +OpenAI Response API SSE events. Used by the streaming event
8 +processors in serving.py.
9 +"""
10 +
11 +import json
12 +from dataclasses import dataclass
13 +from typing import Final
14 +
15 +from openai.types.responses import (
16 + ResponseCodeInterpreterCallCodeDeltaEvent,
17 + ResponseCodeInterpreterCallCodeDoneEvent,
18 + ResponseCodeInterpreterCallCompletedEvent,
19 + ResponseCodeInterpreterCallInProgressEvent,
20 + ResponseCodeInterpreterCallInterpretingEvent,
21 + ResponseCodeInterpreterToolCallParam,
22 + ResponseContentPartAddedEvent,
23 + ResponseContentPartDoneEvent,
24 + ResponseFunctionCallArgumentsDeltaEvent,
25 + ResponseFunctionCallArgumentsDoneEvent,
26 + ResponseFunctionToolCall,
27 + ResponseFunctionWebSearch,
28 + ResponseMcpCallArgumentsDeltaEvent,
29 + ResponseMcpCallArgumentsDoneEvent,
30 + ResponseMcpCallCompletedEvent,
31 + ResponseMcpCallInProgressEvent,
32 + ResponseOutputItemAddedEvent,
33 + ResponseOutputItemDoneEvent,
34 + ResponseOutputMessage,
35 + ResponseOutputText,
36 + ResponseReasoningItem,
37 + ResponseReasoningTextDeltaEvent,
38 + ResponseReasoningTextDoneEvent,
39 + ResponseTextDeltaEvent,
40 + ResponseTextDoneEvent,
41 + ResponseWebSearchCallCompletedEvent,
42 + ResponseWebSearchCallInProgressEvent,
43 + ResponseWebSearchCallSearchingEvent,
44 + response_function_web_search,
45 +)
46 +from openai.types.responses.response_output_item import McpCall
47 +from openai.types.responses.response_reasoning_item import (
48 + Content as ResponseReasoningTextContent,
49 +)
50 +
51 +from vllm.entrypoints.mcp.tool_server import ToolServer
52 +from vllm.entrypoints.openai.responses.context import StreamingHarmonyContext
53 +from vllm.entrypoints.openai.responses.protocol import (
54 + ResponseReasoningPartAddedEvent,
55 + ResponseReasoningPartDoneEvent,
56 + StreamingResponsesResponse,
57 +)
58 +from vllm.utils import random_uuid
59 +
60 +TOOL_NAME_TO_MCP_SERVER_LABEL: Final[dict[str, str]] = {
61 + "python": "code_interpreter",
62 + "container": "container",
63 + "browser": "web_search_preview",
64 +}
65 +
66 +
67 +@dataclass
68 +class HarmonyStreamingState:
69 + """Mutable state for harmony streaming event processing."""
70 +
71 + current_content_index: int = -1
72 + current_output_index: int = 0
73 + current_item_id: str = ""
74 + sent_output_item_added: bool = False
75 + is_first_function_call_delta: bool = False
76 +
77 + def reset_for_new_item(self) -> None:
78 + """Reset state when expecting a new output item."""
79 + self.current_output_index += 1
80 + self.sent_output_item_added = False
81 + self.is_first_function_call_delta = False
82 +
83 +
84 +def is_mcp_tool_by_namespace(recipient: str | None) -> bool:
85 + """
86 + Determine if a tool call is an MCP tool based on recipient prefix.
87 +
88 + - Tools starting with "functions." are function calls
89 + - Everything else is an MCP tool
90 + """
91 + if recipient is None:
92 + return False
93 +
94 + # Function calls have "functions." prefix
95 + # Everything else is an MCP tool
96 + return not recipient.startswith("functions.")
97 +
98 +
99 +def emit_function_call_done_events(
100 + previous_item,
101 + state: HarmonyStreamingState,
102 +) -> list[StreamingResponsesResponse]:
103 + """Emit events when a function call completes."""
104 + function_name = previous_item.recipient[len("functions.") :]
105 + events: list[StreamingResponsesResponse] = []
106 + events.append(
107 + ResponseFunctionCallArgumentsDoneEvent(
108 + type="response.function_call_arguments.done",
109 + arguments=previous_item.content[0].text,
110 + name=function_name,
111 + item_id=state.current_item_id,
112 + output_index=state.current_output_index,
113 + sequence_number=-1,
114 + )
115 + )
116 + function_call_item = ResponseFunctionToolCall(
117 + type="function_call",
118 + arguments=previous_item.content[0].text,
119 + name=function_name,
120 + item_id=state.current_item_id,
121 + output_index=state.current_output_index,
122 + sequence_number=-1,
123 + call_id=f"fc_{random_uuid()}",
124 + status="completed",
125 + )
126 + events.append(
127 + ResponseOutputItemDoneEvent(
128 + type="response.output_item.done",
129 + sequence_number=-1,
130 + output_index=state.current_output_index,
131 + item=function_call_item,
132 + )
133 + )
134 + return events
135 +
136 +
137 +def emit_mcp_call_done_events(
138 + previous_item,
139 + state: HarmonyStreamingState,
140 +) -> list[StreamingResponsesResponse]:
141 + """Emit events when an MCP tool call completes."""
142 + server_label = TOOL_NAME_TO_MCP_SERVER_LABEL.get(
143 + previous_item.recipient, previous_item.recipient
144 + )
145 + events: list[StreamingResponsesResponse] = []
146 + events.append(
147 + ResponseMcpCallArgumentsDoneEvent(
148 + type="response.mcp_call_arguments.done",
149 + arguments=previous_item.content[0].text,
150 + name=previous_item.recipient,
151 + item_id=state.current_item_id,
152 + output_index=state.current_output_index,
153 + sequence_number=-1,
154 + )
155 + )
156 + events.append(
157 + ResponseMcpCallCompletedEvent(
158 + type="response.mcp_call.completed",
159 + sequence_number=-1,
160 + output_index=state.current_output_index,
161 + item_id=state.current_item_id,
162 + )
163 + )
164 + events.append(
165 + ResponseOutputItemDoneEvent(
166 + type="response.output_item.done",
167 + sequence_number=-1,
168 + output_index=state.current_output_index,
169 + item=McpCall(
170 + type="mcp_call",
171 + arguments=previous_item.content[0].text,
172 + name=previous_item.recipient,
173 + id=state.current_item_id,
174 + server_label=server_label,
175 + status="completed",
176 + ),
177 + )
178 + )
179 + return events
180 +
181 +
182 +def emit_reasoning_done_events(
183 + previous_item,
184 + state: HarmonyStreamingState,
185 +) -> list[StreamingResponsesResponse]:
186 + """Emit events when a reasoning (analysis) item completes."""
187 + content = ResponseReasoningTextContent(
188 + text=previous_item.content[0].text,
189 + type="reasoning_text",
190 + )
191 + reasoning_item = ResponseReasoningItem(
192 + type="reasoning",
193 + content=[content],
194 + status="completed",
195 + id=state.current_item_id,
196 + summary=[],
197 + )
198 + events: list[StreamingResponsesResponse] = []
199 + events.append(
200 + ResponseReasoningTextDoneEvent(
201 + type="response.reasoning_text.done",
202 + item_id=state.current_item_id,
203 + sequence_number=-1,
204 + output_index=state.current_output_index,
205 + content_index=state.current_content_index,
206 + text=previous_item.content[0].text,
207 + )
208 + )
209 + events.append(
210 + ResponseReasoningPartDoneEvent(
211 + type="response.reasoning_part.done",
212 + sequence_number=-1,
213 + item_id=state.current_item_id,
214 + output_index=state.current_output_index,
215 + content_index=state.current_content_index,
216 + part=content,
217 + )
218 + )
219 + events.append(
220 + ResponseOutputItemDoneEvent(
221 + type="response.output_item.done",
222 + sequence_number=-1,
223 + output_index=state.current_output_index,
224 + item=reasoning_item,
225 + )
226 + )
227 + return events
228 +
229 +
230 +def emit_text_output_done_events(
231 + previous_item,
232 + state: HarmonyStreamingState,
233 +) -> list[StreamingResponsesResponse]:
234 + """Emit events when a final text output item completes."""
235 + text_content = ResponseOutputText(
236 + type="output_text",
237 + text=previous_item.content[0].text,
238 + annotations=[],
239 + )
240 + events: list[StreamingResponsesResponse] = []
241 + events.append(
242 + ResponseTextDoneEvent(
243 + type="response.output_text.done",
244 + sequence_number=-1,
245 + output_index=state.current_output_index,
246 + content_index=state.current_content_index,
247 + text=previous_item.content[0].text,
248 + logprobs=[],
249 + item_id=state.current_item_id,
250 + )
251 + )
252 + events.append(
253 + ResponseContentPartDoneEvent(
254 + type="response.content_part.done",
255 + sequence_number=-1,
256 + item_id=state.current_item_id,
257 + output_index=state.current_output_index,
258 + content_index=state.current_content_index,
259 + part=text_content,
260 + )
261 + )
262 + events.append(
263 + ResponseOutputItemDoneEvent(
264 + type="response.output_item.done",
265 + sequence_number=-1,
266 + output_index=state.current_output_index,
267 + item=ResponseOutputMessage(
268 + id=state.current_item_id,
269 + type="message",
270 + role="assistant",
271 + content=[text_content],
272 + status="completed",
273 + ),
274 + )
275 + )
276 + return events
277 +
278 +
279 +def emit_previous_item_done_events(
280 + previous_item,
281 + state: HarmonyStreamingState,
282 +) -> list[StreamingResponsesResponse]:
283 + """Emit done events for the previous item when expecting a new start."""
284 + if previous_item.recipient is not None:
285 + # Deal with tool call
286 + if previous_item.recipient.startswith("functions."):
287 + return emit_function_call_done_events(previous_item, state)
288 + elif (
289 + is_mcp_tool_by_namespace(previous_item.recipient)
290 + and state.current_item_id is not None
291 + and state.current_item_id.startswith("mcp_")
292 + ):
293 + return emit_mcp_call_done_events(previous_item, state)
294 + elif previous_item.channel == "analysis":
295 + return emit_reasoning_done_events(previous_item, state)
296 + elif previous_item.channel == "final":
297 + return emit_text_output_done_events(previous_item, state)
298 + return []
299 +
300 +
301 +def emit_final_channel_delta_events(
302 + ctx: StreamingHarmonyContext,
303 + state: HarmonyStreamingState,
304 +) -> list[StreamingResponsesResponse]:
305 + """Emit events for final channel text delta streaming."""
306 + events: list[StreamingResponsesResponse] = []
307 + if not state.sent_output_item_added:
308 + state.sent_output_item_added = True
309 + state.current_item_id = f"msg_{random_uuid()}"
310 + events.append(
311 + ResponseOutputItemAddedEvent(
312 + type="response.output_item.added",
313 + sequence_number=-1,
314 + output_index=state.current_output_index,
315 + item=ResponseOutputMessage(
316 + id=state.current_item_id,
317 + type="message",
318 + role="assistant",
319 + content=[],
320 + status="in_progress",
321 + ),
322 + )
323 + )
324 + state.current_content_index += 1
325 + events.append(
326 + ResponseContentPartAddedEvent(
327 + type="response.content_part.added",
328 + sequence_number=-1,
329 + output_index=state.current_output_index,
330 + item_id=state.current_item_id,
331 + content_index=state.current_content_index,
332 + part=ResponseOutputText(
333 + type="output_text",
334 + text="",
335 + annotations=[],
336 + logprobs=[],
337 + ),
338 + )
339 + )
340 + events.append(
341 + ResponseTextDeltaEvent(
342 + type="response.output_text.delta",
343 + sequence_number=-1,
344 + content_index=state.current_content_index,
345 + output_index=state.current_output_index,
346 + item_id=state.current_item_id,
347 + delta=ctx.last_content_delta,
348 + # TODO, use logprobs from ctx.last_request_output
349 + logprobs=[],
350 + )
351 + )
352 + return events
353 +
354 +
355 +def emit_analysis_channel_delta_events(
356 + ctx: StreamingHarmonyContext,
357 + state: HarmonyStreamingState,
358 +) -> list[StreamingResponsesResponse]:
359 + """Emit events for analysis channel reasoning delta streaming."""
360 + events: list[StreamingResponsesResponse] = []
361 + if not state.sent_output_item_added:
362 + state.sent_output_item_added = True
363 + state.current_item_id = f"msg_{random_uuid()}"
364 + events.append(
365 + ResponseOutputItemAddedEvent(
366 + type="response.output_item.added",
367 + sequence_number=-1,
368 + output_index=state.current_output_index,
369 + item=ResponseReasoningItem(
370 + type="reasoning",
371 + id=state.current_item_id,
372 + summary=[],
373 + status="in_progress",
374 + ),
375 + )
376 + )
377 + state.current_content_index += 1
378 + events.append(
379 + ResponseReasoningPartAddedEvent(
380 + type="response.reasoning_part.added",
381 + sequence_number=-1,
382 + output_index=state.current_output_index,
383 + item_id=state.current_item_id,
384 + content_index=state.current_content_index,
385 + part=ResponseReasoningTextContent(
386 + text="",
387 + type="reasoning_text",
388 + ),
389 + )
390 + )
391 + events.append(
392 + ResponseReasoningTextDeltaEvent(
393 + type="response.reasoning_text.delta",
394 + item_id=state.current_item_id,
395 + output_index=state.current_output_index,
396 + content_index=state.current_content_index,
397 + delta=ctx.last_content_delta,
398 + sequence_number=-1,
399 + )
400 + )
401 + return events
402 +
403 +
404 +def emit_mcp_tool_delta_events(
405 + ctx: StreamingHarmonyContext,
406 + state: HarmonyStreamingState,
407 + recipient: str,
408 +) -> list[StreamingResponsesResponse]:
409 + """Emit events for MCP tool delta streaming."""
410 + server_label = TOOL_NAME_TO_MCP_SERVER_LABEL.get(recipient, recipient)
411 + events: list[StreamingResponsesResponse] = []
412 + if not state.sent_output_item_added:
413 + state.sent_output_item_added = True
414 + state.current_item_id = f"mcp_{random_uuid()}"
415 + events.append(
416 + ResponseOutputItemAddedEvent(
417 + type="response.output_item.added",
418 + sequence_number=-1,
419 + output_index=state.current_output_index,
420 + item=McpCall(
421 + type="mcp_call",
422 + id=state.current_item_id,
423 + name=recipient,
424 + arguments="",
425 + server_label=server_label,
426 + status="in_progress",
427 + ),
428 + )
429 + )
430 + events.append(
431 + ResponseMcpCallInProgressEvent(
432 + type="response.mcp_call.in_progress",
433 + sequence_number=-1,
434 + output_index=state.current_output_index,
435 + item_id=state.current_item_id,
436 + )
437 + )
438 + events.append(
439 + ResponseMcpCallArgumentsDeltaEvent(
440 + type="response.mcp_call_arguments.delta",
441 + sequence_number=-1,
442 + output_index=state.current_output_index,
443 + item_id=state.current_item_id,
444 + delta=ctx.last_content_delta,
445 + )
446 + )
447 + return events
448 +
449 +
450 +def emit_code_interpreter_delta_events(
451 + ctx: StreamingHarmonyContext,
452 + state: HarmonyStreamingState,
453 +) -> list[StreamingResponsesResponse]:
454 + """Emit events for code interpreter delta streaming."""
455 + events: list[StreamingResponsesResponse] = []
456 + if not state.sent_output_item_added:
457 + state.sent_output_item_added = True
458 + state.current_item_id = f"tool_{random_uuid()}"
459 + events.append(
460 + ResponseOutputItemAddedEvent(
461 + type="response.output_item.added",
462 + sequence_number=-1,
463 + output_index=state.current_output_index,
464 + item=ResponseCodeInterpreterToolCallParam(
465 + type="code_interpreter_call",
466 + id=state.current_item_id,
467 + code=None,
468 + container_id="auto",
469 + outputs=None,
470 + status="in_progress",
471 + ),
472 + )
473 + )
474 + events.append(
475 + ResponseCodeInterpreterCallInProgressEvent(
476 + type="response.code_interpreter_call.in_progress",
477 + sequence_number=-1,
478 + output_index=state.current_output_index,
479 + item_id=state.current_item_id,
480 + )
481 + )
482 + events.append(
483 + ResponseCodeInterpreterCallCodeDeltaEvent(
484 + type="response.code_interpreter_call_code.delta",
485 + sequence_number=-1,
486 + output_index=state.current_output_index,
487 + item_id=state.current_item_id,
488 + delta=ctx.last_content_delta,
489 + )
490 + )
491 + return events
492 +
493 +
494 +def emit_mcp_prefix_delta_events(
495 + ctx: StreamingHarmonyContext,
496 + state: HarmonyStreamingState,
497 +) -> list[StreamingResponsesResponse]:
498 + """Emit events for MCP prefix (mcp.*) delta streaming."""
499 + events: list[StreamingResponsesResponse] = []
500 + if not state.sent_output_item_added:
501 + state.sent_output_item_added = True
502 + state.current_item_id = f"mcp_{random_uuid()}"
503 + mcp_name = ctx.parser.current_recipient[len("mcp.") :]
504 +
505 + events.append(
506 + ResponseOutputItemAddedEvent(
507 + type="response.output_item.added",
508 + sequence_number=-1,
509 + output_index=state.current_output_index,
510 + item=McpCall(
511 + type="mcp_call",
512 + id=state.current_item_id,
513 + name=mcp_name,
514 + arguments="",
515 + server_label=mcp_name,
516 + status="in_progress",
517 + ),
518 + )
519 + )
520 + events.append(
521 + ResponseMcpCallInProgressEvent(
522 + type="response.mcp_call.in_progress",
523 + sequence_number=-1,
524 + output_index=state.current_output_index,
525 + item_id=state.current_item_id,
526 + )
527 + )
528 +
529 + events.append(
530 + ResponseMcpCallArgumentsDeltaEvent(
531 + type="response.mcp_call_arguments.delta",
532 + sequence_number=-1,
533 + output_index=state.current_output_index,
534 + item_id=state.current_item_id,
535 + delta=ctx.last_content_delta,
536 + )
537 + )
538 + return events
539 +
540 +
541 +def emit_function_call_delta_events(
542 + ctx: StreamingHarmonyContext,
543 + state: HarmonyStreamingState,
544 +) -> list[StreamingResponsesResponse]:
545 + """Emit events for developer function calls on commentary channel."""
546 + if not (
547 + ctx.parser.current_channel == "commentary"
548 + and ctx.parser.current_recipient
549 + and ctx.parser.current_recipient.startswith("functions.")
550 + ):
551 + return []
552 +
553 + events: list[StreamingResponsesResponse] = []
554 + if state.is_first_function_call_delta is False:
555 + state.is_first_function_call_delta = True
556 + fc_name = ctx.parser.current_recipient[len("functions.") :]
557 + state.current_item_id = f"fc_{random_uuid()}"
558 + tool_call_item = ResponseFunctionToolCall(
559 + name=fc_name,
560 + type="function_call",
561 + id=state.current_item_id,
562 + call_id=f"call_{random_uuid()}",
563 + arguments="",
564 + status="in_progress",
565 + )
566 + events.append(
567 + ResponseOutputItemAddedEvent(
568 + type="response.output_item.added",
569 + sequence_number=-1,
570 + output_index=state.current_output_index,
571 + item=tool_call_item,
572 + )
573 + )
574 + # Always emit the delta (including on first call)
575 + events.append(
576 + ResponseFunctionCallArgumentsDeltaEvent(
577 + item_id=state.current_item_id,
578 + delta=ctx.last_content_delta,
579 + output_index=state.current_output_index,
580 + sequence_number=-1,
581 + type="response.function_call_arguments.delta",
582 + )
583 + )
584 + return events
585 +
586 +
587 +def emit_content_delta_events(
588 + ctx: StreamingHarmonyContext,
589 + state: HarmonyStreamingState,
590 +) -> list[StreamingResponsesResponse]:
591 + """Emit events for content delta streaming based on channel type."""
592 + if not ctx.last_content_delta:
593 + return []
594 +
595 + if ctx.parser.current_channel == "final" and ctx.parser.current_recipient is None:
596 + return emit_final_channel_delta_events(ctx, state)
597 + elif (
598 + ctx.parser.current_channel == "analysis"
599 + and ctx.parser.current_recipient is None
600 + ):
601 + return emit_analysis_channel_delta_events(ctx, state)
602 + # built-in tools will be triggered on the analysis channel
603 + # However, occasionally built-in tools will
604 + # still be output to commentary.
605 + elif (
606 + ctx.parser.current_channel == "commentary"
607 + or ctx.parser.current_channel == "analysis"
608 + ) and ctx.parser.current_recipient is not None:
609 + recipient = ctx.parser.current_recipient
610 + # Check for function calls first - they have their own event handling
611 + if recipient.startswith("functions."):
612 + return emit_function_call_delta_events(ctx, state)
613 + if is_mcp_tool_by_namespace(recipient):
614 + return emit_mcp_tool_delta_events(ctx, state, recipient)
615 + else:
616 + return emit_code_interpreter_delta_events(ctx, state)
617 + elif (
618 + (
619 + ctx.parser.current_channel == "commentary"
620 + or ctx.parser.current_channel == "analysis"
621 + )
622 + and ctx.parser.current_recipient is not None
623 + and ctx.parser.current_recipient.startswith("mcp.")
624 + ):
625 + return emit_mcp_prefix_delta_events(ctx, state)
626 +
627 + return []
628 +
629 +
630 +def emit_browser_tool_events(
631 + previous_item,
632 + state: HarmonyStreamingState,
633 +) -> list[StreamingResponsesResponse]:
634 + """Emit events for browser tool calls (web search)."""
635 + function_name = previous_item.recipient[len("browser.") :]
636 + parsed_args = json.loads(previous_item.content[0].text)
637 + action = None
638 +
639 + if function_name == "search":
640 + action = response_function_web_search.ActionSearch(
641 + type="search",
642 + query=parsed_args["query"],
643 + )
644 + elif function_name == "open":
645 + action = response_function_web_search.ActionOpenPage(
646 + type="open_page",
647 + # TODO: translate to url
648 + url=f"cursor:{parsed_args.get('cursor', '')}",
649 + )
650 + elif function_name == "find":
651 + action = response_function_web_search.ActionFind(
652 + type="find",
653 + pattern=parsed_args["pattern"],
654 + # TODO: translate to url
655 + url=f"cursor:{parsed_args.get('cursor', '')}",
656 + )
657 + else:
658 + raise ValueError(f"Unknown function name: {function_name}")
659 +
660 + state.current_item_id = f"tool_{random_uuid()}"
661 + events: list[StreamingResponsesResponse] = []
662 + events.append(
663 + ResponseOutputItemAddedEvent(
664 + type="response.output_item.added",
665 + sequence_number=-1,
666 + output_index=state.current_output_index,
667 + item=response_function_web_search.ResponseFunctionWebSearch(
668 + # TODO: generate a unique id for web search call
669 + type="web_search_call",
670 + id=state.current_item_id,
671 + action=action,
672 + status="in_progress",
673 + ),
674 + )
675 + )
676 + events.append(
677 + ResponseWebSearchCallInProgressEvent(
678 + type="response.web_search_call.in_progress",
679 + sequence_number=-1,
680 + output_index=state.current_output_index,
681 + item_id=state.current_item_id,
682 + )
683 + )
684 + events.append(
685 + ResponseWebSearchCallSearchingEvent(
686 + type="response.web_search_call.searching",
687 + sequence_number=-1,
688 + output_index=state.current_output_index,
689 + item_id=state.current_item_id,
690 + )
691 + )
692 + # enqueue
693 + events.append(
694 + ResponseWebSearchCallCompletedEvent(
695 + type="response.web_search_call.completed",
696 + sequence_number=-1,
697 + output_index=state.current_output_index,
698 + item_id=state.current_item_id,
699 + )
700 + )
701 + events.append(
702 + ResponseOutputItemDoneEvent(
703 + type="response.output_item.done",
704 + sequence_number=-1,
705 + output_index=state.current_output_index,
706 + item=ResponseFunctionWebSearch(
707 + type="web_search_call",
708 + id=state.current_item_id,
709 + action=action,
710 + status="completed",
711 + ),
712 + )
713 + )
714 + return events
715 +
716 +
717 +def emit_mcp_tool_completion_events(
718 + previous_item,
719 + state: HarmonyStreamingState,
720 +) -> list[StreamingResponsesResponse]:
721 + """Emit events when an MCP tool completes during assistant action turn."""
722 + recipient = previous_item.recipient
723 + server_label = TOOL_NAME_TO_MCP_SERVER_LABEL.get(recipient, recipient)
724 + events: list[StreamingResponsesResponse] = []
725 + events.append(
726 + ResponseMcpCallArgumentsDoneEvent(
727 + type="response.mcp_call_arguments.done",
728 + sequence_number=-1,
729 + output_index=state.current_output_index,
730 + item_id=state.current_item_id,
731 + arguments=previous_item.content[0].text,
732 + name=recipient,
733 + )
734 + )
735 + events.append(
736 + ResponseMcpCallCompletedEvent(
737 + type="response.mcp_call.completed",
738 + sequence_number=-1,
739 + output_index=state.current_output_index,
740 + item_id=state.current_item_id,
741 + )
742 + )
743 + events.append(
744 + ResponseOutputItemDoneEvent(
745 + type="response.output_item.done",
746 + sequence_number=-1,
747 + output_index=state.current_output_index,
748 + item=McpCall(
749 + type="mcp_call",
750 + id=state.current_item_id,
751 + name=recipient,
752 + arguments=previous_item.content[0].text,
753 + server_label=server_label,
754 + status="completed",
755 + ),
756 + )
757 + )
758 + return events
759 +
760 +
761 +def emit_code_interpreter_completion_events(
762 + previous_item,
763 + state: HarmonyStreamingState,
764 +) -> list[StreamingResponsesResponse]:
765 + """Emit events when code interpreter completes."""
766 + events: list[StreamingResponsesResponse] = []
767 + events.append(
768 + ResponseCodeInterpreterCallCodeDoneEvent(
769 + type="response.code_interpreter_call_code.done",
770 + sequence_number=-1,
771 + output_index=state.current_output_index,
772 + item_id=state.current_item_id,
773 + code=previous_item.content[0].text,
774 + )
775 + )
776 + events.append(
777 + ResponseCodeInterpreterCallInterpretingEvent(
778 + type="response.code_interpreter_call.interpreting",
779 + sequence_number=-1,
780 + output_index=state.current_output_index,
781 + item_id=state.current_item_id,
782 + )
783 + )
784 + events.append(
785 + ResponseCodeInterpreterCallCompletedEvent(
786 + type="response.code_interpreter_call.completed",
787 + sequence_number=-1,
788 + output_index=state.current_output_index,
789 + item_id=state.current_item_id,
790 + )
791 + )
792 + events.append(
793 + ResponseOutputItemDoneEvent(
794 + type="response.output_item.done",
795 + sequence_number=-1,
796 + output_index=state.current_output_index,
797 + item=ResponseCodeInterpreterToolCallParam(
798 + type="code_interpreter_call",
799 + id=state.current_item_id,
800 + code=previous_item.content[0].text,
801 + container_id="auto",
802 + outputs=[],
803 + status="completed",
804 + ),
805 + )
806 + )
807 + return events
808 +
809 +
810 +def emit_mcp_prefix_completion_events(
811 + previous_item,
812 + state: HarmonyStreamingState,
813 +) -> list[StreamingResponsesResponse]:
814 + """Emit events when an MCP prefix tool (mcp.*) completes."""
815 + mcp_name = previous_item.recipient[len("mcp.") :]
816 + events: list[StreamingResponsesResponse] = []
817 + events.append(
818 + ResponseMcpCallArgumentsDoneEvent(
819 + type="response.mcp_call_arguments.done",
820 + sequence_number=-1,
821 + output_index=state.current_output_index,
822 + item_id=state.current_item_id,
823 + arguments=previous_item.content[0].text,
824 + name=mcp_name,
825 + )
826 + )
827 + events.append(
828 + ResponseMcpCallCompletedEvent(
829 + type="response.mcp_call.completed",
830 + sequence_number=-1,
831 + output_index=state.current_output_index,
832 + item_id=state.current_item_id,
833 + )
834 + )
835 + events.append(
836 + ResponseOutputItemDoneEvent(
837 + type="response.output_item.done",
838 + sequence_number=-1,
839 + output_index=state.current_output_index,
840 + item=McpCall(
841 + type="mcp_call",
842 + id=state.current_item_id,
843 + name=mcp_name,
844 + arguments=previous_item.content[0].text,
845 + server_label=mcp_name,
846 + status="completed",
847 + ),
848 + )
849 + )
850 + return events
851 +
852 +
853 +def emit_tool_action_events(
854 + ctx: StreamingHarmonyContext,
855 + state: HarmonyStreamingState,
856 + tool_server: ToolServer | None,
857 +) -> list[StreamingResponsesResponse]:
858 + """Emit events for tool action turn."""
859 + if not ctx.is_assistant_action_turn() or len(ctx.parser.messages) == 0:
860 + return []
861 +
862 + events: list[StreamingResponsesResponse] = []
863 + previous_item = ctx.parser.messages[-1]
864 +
865 + # Handle browser tool
866 + if (
867 + tool_server is not None
868 + and tool_server.has_tool("browser")
869 + and previous_item.recipient is not None
870 + and previous_item.recipient.startswith("browser.")
871 + ):
872 + events.extend(emit_browser_tool_events(previous_item, state))
873 +
874 + # Handle tool completion
875 + if (
876 + tool_server is not None
877 + and previous_item.recipient is not None
878 + and state.current_item_id is not None
879 + and state.sent_output_item_added
880 + ):
881 + recipient = previous_item.recipient
882 + # Handle MCP prefix tool completion first
883 + if recipient.startswith("mcp."):
884 + events.extend(emit_mcp_prefix_completion_events(previous_item, state))
885 + else:
886 + # Handle other MCP tool and code interpreter completion
887 + is_mcp_tool = is_mcp_tool_by_namespace(
888 + recipient
889 + ) and state.current_item_id.startswith("mcp_")
890 + if is_mcp_tool:
891 + events.extend(emit_mcp_tool_completion_events(previous_item, state))
892 + else:
893 + events.extend(
894 + emit_code_interpreter_completion_events(previous_item, state)
895 + )
896 +
897 + return events