@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 186ea22efe

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 17:45 UTC

[Misc][Harmony] Move Responses API only harmony utils to responses/harmony.py (#35339)

Signed-off-by: sfeng33 <4florafeng@gmail.com>

Flora Feng committed Feb 26, 2026 at 01:35 UTC 186ea22efefd2c6f4f9b7fcb657bd00f50cb465a

6 files changed +1040 -990

tests/entrypoints/openai/parser/test_harmony_utils.py

+7 -460

@@ -2,13 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 import pytest

5 -from openai.types.responses import (

6 - ResponseFunctionToolCall,

7 - ResponseOutputMessage,

8 - ResponseReasoningItem,

9 -)

10 -from openai.types.responses.response_output_item import McpCall

11 -from openai_harmony import Author, Message, Role, TextContent

5 +from openai_harmony import Message, Role

7 from tests.entrypoints.openai.utils import verify_harmony_messages

8 from vllm.entrypoints.openai.parser.harmony_utils import (

@@ -18,20 +12,21 @@ from vllm.entrypoints.openai.parser.harmony_utils import (

12 has_custom_tools,

13 parse_chat_input_to_harmony_message,

14 parse_chat_output,

21 - parse_input_to_harmony_message,

22 - parse_output_message,

15 +)

16 +from vllm.entrypoints.openai.responses.harmony import (

17 + response_previous_input_to_harmony,

18 )

21 class TestCommonParseInputToHarmonyMessage:

22 """

23 Tests for scenarios that are common to both Chat Completion

29 - parse_chat_input_to_harmony_message and Responsees API

30 - parse_input_to_harmony_message functions.

24 + parse_chat_input_to_harmony_message and Responses API

25 + response_previous_input_to_harmony functions.

26 """

28 @pytest.fixture(

34 - params=[parse_chat_input_to_harmony_message, parse_input_to_harmony_message]

29 + params=[parse_chat_input_to_harmony_message, response_previous_input_to_harmony]

30 )

31 def parse_function(self, request):

32 return request.param

@@ -216,81 +211,6 @@ class TestCommonParseInputToHarmonyMessage:

211 assert messages[0].content[1].text == "actual text"

212

213

219 -class TestParseInputToHarmonyMessage:

220 - """

221 - Tests for scenarios that are specific to the Responses API

222 - parse_input_to_harmony_message function.

223 - """

224 -

225 - def test_message_with_empty_content(self):

226 - """Test parsing message with empty string content."""

227 - chat_msg = {

228 - "role": "user",

229 - "content": "",

230 - }

231 -

232 - messages = parse_input_to_harmony_message(chat_msg)

233 -

234 - assert len(messages) == 1

235 - assert messages[0].content[0].text == ""

236 -

237 - def test_tool_message_with_string_content(self):

238 - """Test parsing tool message with string content."""

239 - chat_msg = {

240 - "role": "tool",

241 - "name": "get_weather",

242 - "content": "The weather in San Francisco is sunny, 72°F",

243 - }

244 -

245 - messages = parse_input_to_harmony_message(chat_msg)

246 -

247 - assert len(messages) == 1

248 - assert messages[0].author.role == Role.TOOL

249 - assert messages[0].author.name == "functions.get_weather"

250 - assert (

251 - messages[0].content[0].text == "The weather in San Francisco is sunny, 72°F"

252 - )

253 - assert messages[0].channel == "commentary"

254 -

255 - def test_tool_message_with_array_content(self):

256 - """Test parsing tool message with array content."""

257 - chat_msg = {

258 - "role": "tool",

259 - "name": "search_results",

260 - "content": [

261 - {"type": "text", "text": "Result 1: "},

262 - {"type": "text", "text": "Result 2: "},

263 - {

264 - "type": "image",

265 - "url": "http://example.com/img.png",

266 - }, # Should be ignored

267 - {"type": "text", "text": "Result 3"},

268 - ],

269 - }

270 -

271 - messages = parse_input_to_harmony_message(chat_msg)

272 -

273 - assert len(messages) == 1

274 - assert messages[0].author.role == Role.TOOL

275 - assert messages[0].author.name == "functions.search_results"

276 - assert messages[0].content[0].text == "Result 1: Result 2: Result 3"

277 -

278 - def test_tool_message_with_empty_content(self):

279 - """Test parsing tool message with None content."""

280 - chat_msg = {

281 - "role": "tool",

282 - "name": "empty_tool",

283 - "content": None,

284 - }

285 -

286 - messages = parse_input_to_harmony_message(chat_msg)

287 -

288 - assert len(messages) == 1

289 - assert messages[0].author.role == Role.TOOL

290 - assert messages[0].author.name == "functions.empty_tool"

291 - assert messages[0].content[0].text == ""

292 -

293 -

214 class TestParseChatInputToHarmonyMessage:

215 """

216 Tests for scenarios that are specific to the Chat Completion API

@@ -888,200 +808,6 @@ class TestParseChatOutput:

808 assert final_content == "Let me look that up.\nThe answer is 42."

809

810

891 -class TestParseOutputMessage:

892 - """Tests for parse_output_message function."""

893 -

894 - def test_commentary_with_no_recipient_creates_message(self):

895 - """Test that commentary with recipient=None (preambles) creates message items.

896 -

897 - Per Harmony format, preambles are intended to be shown to end-users,

898 - unlike analysis channel content which is hidden reasoning.

899 - See: https://cookbook.openai.com/articles/openai-harmony

900 - """

901 - message = Message.from_role_and_content(

902 - Role.ASSISTANT, "I will now search for the weather information."

903 - )

904 - message = message.with_channel("commentary")

905 - # recipient is None by default, representing a preamble

906 -

907 - output_items = parse_output_message(message)

908 -

909 - assert len(output_items) == 1

910 - assert isinstance(output_items[0], ResponseOutputMessage)

911 - assert output_items[0].type == "message"

912 - assert output_items[0].role == "assistant"

913 - assert output_items[0].status == "completed"

914 - assert len(output_items[0].content) == 1

915 - assert output_items[0].content[0].type == "output_text"

916 - assert (

917 - output_items[0].content[0].text

918 - == "I will now search for the weather information."

919 - )

920 -

921 - def test_commentary_with_function_recipient_creates_function_call(self):

922 - """Test commentary with recipient='functions.X' creates function calls."""

923 - message = Message.from_role_and_content(

924 - Role.ASSISTANT, '{"location": "San Francisco", "units": "celsius"}'

925 - )

926 - message = message.with_channel("commentary")

927 - message = message.with_recipient("functions.get_weather")

928 -

929 - output_items = parse_output_message(message)

930 -

931 - assert len(output_items) == 1

932 - assert isinstance(output_items[0], ResponseFunctionToolCall)

933 - assert output_items[0].type == "function_call"

934 - assert output_items[0].name == "get_weather"

935 - assert (

936 - output_items[0].arguments

937 - == '{"location": "San Francisco", "units": "celsius"}'

938 - )

939 - assert output_items[0].call_id.startswith("call_")

940 - assert output_items[0].id.startswith("fc_")

941 -

942 - def test_commentary_with_python_recipient_creates_reasoning(self):

943 - """Test that commentary with recipient='python' creates reasoning items."""

944 - message = Message.from_role_and_content(

945 - Role.ASSISTANT, "import numpy as np\nprint(np.array([1, 2, 3]))"

946 - )

947 - message = message.with_channel("commentary")

948 - message = message.with_recipient("python")

949 -

950 - output_items = parse_output_message(message)

951 -

952 - assert len(output_items) == 1

953 - assert isinstance(output_items[0], ResponseReasoningItem)

954 - assert output_items[0].type == "reasoning"

955 - assert (

956 - output_items[0].content[0].text

957 - == "import numpy as np\nprint(np.array([1, 2, 3]))"

958 - )

959 -

960 - def test_commentary_with_browser_recipient_creates_reasoning(self):

961 - """Test that commentary with recipient='browser' creates reasoning items."""

962 - message = Message.from_role_and_content(

963 - Role.ASSISTANT, "Navigating to the specified URL"

964 - )

965 - message = message.with_channel("commentary")

966 - message = message.with_recipient("browser")

967 -

968 - output_items = parse_output_message(message)

969 -

970 - assert len(output_items) == 1

971 - assert isinstance(output_items[0], ResponseReasoningItem)

972 - assert output_items[0].type == "reasoning"

973 - assert output_items[0].content[0].text == "Navigating to the specified URL"

974 -

975 - def test_commentary_with_container_recipient_creates_reasoning(self):

976 - """Test that commentary with recipient='container' creates reasoning items."""

977 - message = Message.from_role_and_content(

978 - Role.ASSISTANT, "Running command in container"

979 - )

980 - message = message.with_channel("commentary")

981 - message = message.with_recipient("container")

982 -

983 - output_items = parse_output_message(message)

984 -

985 - assert len(output_items) == 1

986 - assert isinstance(output_items[0], ResponseReasoningItem)

987 - assert output_items[0].type == "reasoning"

988 - assert output_items[0].content[0].text == "Running command in container"

989 -

990 - def test_commentary_with_empty_content_and_no_recipient(self):

991 - """Test edge case: empty commentary with recipient=None."""

992 - message = Message.from_role_and_content(Role.ASSISTANT, "")

993 - message = message.with_channel("commentary")

994 -

995 - output_items = parse_output_message(message)

996 -

997 - assert len(output_items) == 1

998 - assert isinstance(output_items[0], ResponseOutputMessage)

999 - assert output_items[0].content[0].text == ""

1000 -

1001 - def test_commentary_with_multiple_contents_and_no_recipient(self):

1002 - """Test multiple content items in commentary with no recipient."""

1003 - contents = [

1004 - TextContent(text="Step 1: Analyze the request"),

1005 - TextContent(text="Step 2: Prepare to call functions"),

1006 - ]

1007 - message = Message.from_role_and_contents(Role.ASSISTANT, contents)

1008 - message = message.with_channel("commentary")

1009 -

1010 - output_items = parse_output_message(message)

1011 -

1012 - # _parse_final_message returns single ResponseOutputMessage with

1013 - # multiple contents

1014 - assert len(output_items) == 1

1015 - assert isinstance(output_items[0], ResponseOutputMessage)

1016 - assert len(output_items[0].content) == 2

1017 - assert output_items[0].content[0].text == "Step 1: Analyze the request"

1018 - assert output_items[0].content[1].text == "Step 2: Prepare to call functions"

1019 -

1020 - def test_commentary_with_multiple_function_calls(self):

1021 - """Test multiple function calls in commentary channel."""

1022 - contents = [

1023 - TextContent(text='{"location": "San Francisco"}'),

1024 - TextContent(text='{"location": "New York"}'),

1025 - ]

1026 - message = Message.from_role_and_contents(Role.ASSISTANT, contents)

1027 - message = message.with_channel("commentary")

1028 - message = message.with_recipient("functions.get_weather")

1029 -

1030 - output_items = parse_output_message(message)

1031 -

1032 - assert len(output_items) == 2

1033 - assert all(isinstance(item, ResponseFunctionToolCall) for item in output_items)

1034 - assert output_items[0].name == "get_weather"

1035 - assert output_items[1].name == "get_weather"

1036 - assert output_items[0].arguments == '{"location": "San Francisco"}'

1037 - assert output_items[1].arguments == '{"location": "New York"}'

1038 -

1039 - def test_commentary_with_unknown_recipient_creates_mcp_call(self):

1040 - """Test that commentary with unknown recipient creates MCP call."""

1041 - message = Message.from_role_and_content(Role.ASSISTANT, '{"arg": "value"}')

1042 - message = message.with_channel("commentary")

1043 - message = message.with_recipient("custom_tool")

1044 -

1045 - output_items = parse_output_message(message)

1046 -

1047 - assert len(output_items) == 1

1048 - assert isinstance(output_items[0], McpCall)

1049 - assert output_items[0].type == "mcp_call"

1050 - assert output_items[0].name == "custom_tool"

1051 - assert output_items[0].server_label == "custom_tool"

1052 -

1053 - def test_analysis_channel_creates_reasoning(self):

1054 - """Test that analysis channel creates reasoning items."""

1055 - message = Message.from_role_and_content(

1056 - Role.ASSISTANT, "Analyzing the problem step by step..."

1057 - )

1058 - message = message.with_channel("analysis")

1059 -

1060 - output_items = parse_output_message(message)

1061 -

1062 - assert len(output_items) == 1

1063 - assert isinstance(output_items[0], ResponseReasoningItem)

1064 - assert output_items[0].type == "reasoning"

1065 - assert (

1066 - output_items[0].content[0].text == "Analyzing the problem step by step..."

1067 - )

1068 -

1069 - def test_non_assistant_message_returns_empty(self):

1070 - """Test that non-assistant messages return empty list.

1071 -

1072 - Per the implementation, tool messages to assistant (e.g., search results)

1073 - are not included in final output to align with OpenAI behavior.

1074 - """

1075 - message = Message.from_author_and_content(

1076 - Author.new(Role.TOOL, "functions.get_weather"),

1077 - "The weather is sunny, 72°F",

1078 - )

1079 -

1080 - output_items = parse_output_message(message)

1081 -

1082 - assert len(output_items) == 0

1083 -

1084 -

811 def test_has_custom_tools() -> None:

812 assert not has_custom_tools(set())

813 assert not has_custom_tools({"web_search_preview", "code_interpreter", "container"})

@@ -1091,185 +817,6 @@ def test_has_custom_tools() -> None:

817 )

818

819

1094 -def test_parse_mcp_call_basic() -> None:

1095 - """Test that MCP calls are parsed with correct type and server_label."""

1096 - message = Message.from_role_and_content(Role.ASSISTANT, '{"path": "/tmp"}')

1097 - message = message.with_recipient("filesystem")

1098 - message = message.with_channel("commentary")

1099 -

1100 - output_items = parse_output_message(message)

1101 -

1102 - assert len(output_items) == 1

1103 - assert isinstance(output_items[0], McpCall)

1104 - assert output_items[0].type == "mcp_call"

1105 - assert output_items[0].name == "filesystem"

1106 - assert output_items[0].server_label == "filesystem"

1107 - assert output_items[0].arguments == '{"path": "/tmp"}'

1108 - assert output_items[0].status == "completed"

1109 -

1110 -

1111 -def test_parse_mcp_call_dotted_recipient() -> None:

1112 - """Test that dotted recipients extract the tool name correctly."""

1113 - message = Message.from_role_and_content(Role.ASSISTANT, '{"cmd": "ls"}')

1114 - message = message.with_recipient("repo_browser.list")

1115 - message = message.with_channel("commentary")

1116 -

1117 - output_items = parse_output_message(message)

1118 -

1119 - assert len(output_items) == 1

1120 - assert isinstance(output_items[0], McpCall)

1121 - assert output_items[0].name == "list"

1122 - assert output_items[0].server_label == "repo_browser"

1123 -

1124 -

1125 -def test_mcp_vs_function_call() -> None:

1126 - """Test that function calls are not parsed as MCP calls."""

1127 - func_message = Message.from_role_and_content(Role.ASSISTANT, '{"arg": "value"}')

1128 - func_message = func_message.with_recipient("functions.my_tool")

1129 - func_message = func_message.with_channel("commentary")

1130 -

1131 - func_items = parse_output_message(func_message)

1132 -

1133 - assert len(func_items) == 1

1134 - assert not isinstance(func_items[0], McpCall)

1135 - assert func_items[0].type == "function_call"

1136 -

1137 -

1138 -def test_mcp_vs_builtin_tools() -> None:

1139 - """Test that built-in tools (python, container) are not parsed as MCP calls."""

1140 - # Test python (built-in tool) - should be reasoning, not MCP

1141 - python_message = Message.from_role_and_content(Role.ASSISTANT, "print('hello')")

1142 - python_message = python_message.with_recipient("python")

1143 - python_message = python_message.with_channel("commentary")

1144 -

1145 - python_items = parse_output_message(python_message)

1146 -

1147 - assert len(python_items) == 1

1148 - assert not isinstance(python_items[0], McpCall)

1149 - assert python_items[0].type == "reasoning"

1150 -

1151 -

1152 -def test_parse_remaining_state_commentary_channel() -> None:

1153 - """Test parse_remaining_state with commentary channel and various recipients."""

1154 - from unittest.mock import Mock

1155 -

1156 - from vllm.entrypoints.openai.parser.harmony_utils import parse_remaining_state

1157 -

1158 - # Test 1: functions.* recipient → should return function tool call

1159 - parser_func = Mock()

1160 - parser_func.current_content = '{"arg": "value"}'

1161 - parser_func.current_role = Role.ASSISTANT

1162 - parser_func.current_channel = "commentary"

1163 - parser_func.current_recipient = "functions.my_tool"

1164 -

1165 - func_items = parse_remaining_state(parser_func)

1166 -

1167 - assert len(func_items) == 1

1168 - assert not isinstance(func_items[0], McpCall)

1169 - assert func_items[0].type == "function_call"

1170 - assert func_items[0].name == "my_tool"

1171 - assert func_items[0].status == "in_progress"

1172 -

1173 - # Test 2: MCP tool (not builtin) → should return MCP call

1174 - parser_mcp = Mock()

1175 - parser_mcp.current_content = '{"path": "/tmp"}'

1176 - parser_mcp.current_role = Role.ASSISTANT

1177 - parser_mcp.current_channel = "commentary"

1178 - parser_mcp.current_recipient = "filesystem"

1179 -

1180 - mcp_items = parse_remaining_state(parser_mcp)

1181 -

1182 - assert len(mcp_items) == 1

1183 - assert isinstance(mcp_items[0], McpCall)

1184 - assert mcp_items[0].type == "mcp_call"

1185 - assert mcp_items[0].name == "filesystem"

1186 - assert mcp_items[0].server_label == "filesystem"

1187 - assert mcp_items[0].status == "in_progress"

1188 -

1189 - # Test 3: Built-in tool (python)

1190 - # should NOT return MCP call, returns reasoning (internal tool interaction)

1191 - parser_builtin = Mock()

1192 - parser_builtin.current_content = "print('hello')"

1193 - parser_builtin.current_role = Role.ASSISTANT

1194 - parser_builtin.current_channel = "commentary"

1195 - parser_builtin.current_recipient = "python"

1196 -

1197 - builtin_items = parse_remaining_state(parser_builtin)

1198 -

1199 - # Built-in tools explicitly return reasoning

1200 - assert len(builtin_items) == 1

1201 - assert not isinstance(builtin_items[0], McpCall)

1202 - assert builtin_items[0].type == "reasoning"

1203 -

1204 - # Test 4: No recipient (preamble) → should return message, not reasoning

1205 - parser_preamble = Mock()

1206 - parser_preamble.current_content = "I'll search for that information now."

1207 - parser_preamble.current_role = Role.ASSISTANT

1208 - parser_preamble.current_channel = "commentary"

1209 - parser_preamble.current_recipient = None

1210 -

1211 - preamble_items = parse_remaining_state(parser_preamble)

1212 -

1213 - assert len(preamble_items) == 1

1214 - assert isinstance(preamble_items[0], ResponseOutputMessage)

1215 - assert preamble_items[0].type == "message"

1216 - assert preamble_items[0].content[0].text == "I'll search for that information now."

1217 - assert preamble_items[0].status == "incomplete" # streaming

1218 -

1219 -

1220 -def test_parse_remaining_state_analysis_channel() -> None:

1221 - """Test parse_remaining_state with analysis channel and various recipients."""

1222 - from unittest.mock import Mock

1223 -

1224 - from vllm.entrypoints.openai.parser.harmony_utils import parse_remaining_state

1225 -

1226 - # Test 1: functions.* recipient → should return function tool call

1227 - parser_func = Mock()

1228 - parser_func.current_content = '{"arg": "value"}'

1229 - parser_func.current_role = Role.ASSISTANT

1230 - parser_func.current_channel = "analysis"

1231 - parser_func.current_recipient = "functions.my_tool"

1232 -

1233 - func_items = parse_remaining_state(parser_func)

1234 -

1235 - assert len(func_items) == 1

1236 - assert not isinstance(func_items[0], McpCall)

1237 - assert func_items[0].type == "function_call"

1238 - assert func_items[0].name == "my_tool"

1239 - assert func_items[0].status == "in_progress"

1240 -

1241 - # Test 2: MCP tool (not builtin) → should return MCP call

1242 - parser_mcp = Mock()

1243 - parser_mcp.current_content = '{"query": "test"}'

1244 - parser_mcp.current_role = Role.ASSISTANT

1245 - parser_mcp.current_channel = "analysis"

1246 - parser_mcp.current_recipient = "database"

1247 -

1248 - mcp_items = parse_remaining_state(parser_mcp)

1249 -

1250 - assert len(mcp_items) == 1

1251 - assert isinstance(mcp_items[0], McpCall)

1252 - assert mcp_items[0].type == "mcp_call"

1253 - assert mcp_items[0].name == "database"

1254 - assert mcp_items[0].server_label == "database"

1255 - assert mcp_items[0].status == "in_progress"

1256 -

1257 - # Test 3: Built-in tool (container)

1258 - # should NOT return MCP call, falls through to reasoning

1259 - parser_builtin = Mock()

1260 - parser_builtin.current_content = "docker run"

1261 - parser_builtin.current_role = Role.ASSISTANT

1262 - parser_builtin.current_channel = "analysis"

1263 - parser_builtin.current_recipient = "container"

1264 -

1265 - builtin_items = parse_remaining_state(parser_builtin)

1266 -

1267 - # Should fall through to reasoning logic

1268 - assert len(builtin_items) == 1

1269 - assert not isinstance(builtin_items[0], McpCall)

1270 - assert builtin_items[0].type == "reasoning"

1271 -

1272 -

820 class TestGetSystemMessage:

821 """Tests for get_system_message channel configuration."""

822

tests/entrypoints/openai/responses/test_harmony_utils.py new

+463

@@ -0,0 +1,463 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Unit tests for vllm.entrypoints.openai.responses.harmony."""

4 +

5 +from openai.types.responses import (

6 + ResponseFunctionToolCall,

7 + ResponseOutputMessage,

8 + ResponseReasoningItem,

9 +)

10 +from openai.types.responses.response_output_item import McpCall

11 +from openai_harmony import Author, Message, Role, TextContent

12 +

13 +from vllm.entrypoints.openai.responses.harmony import (

14 + harmony_to_response_output,

15 + parser_state_to_response_output,

16 + response_previous_input_to_harmony,

17 +)

18 +

19 +

20 +class TestResponsePreviousInputToHarmony:

21 + """

22 + Tests for scenarios that are specific to the Responses API

23 + response_previous_input_to_harmony function.

24 + """

25 +

26 + def test_message_with_empty_content(self):

27 + """Test parsing message with empty string content."""

28 + chat_msg = {

29 + "role": "user",

30 + "content": "",

31 + }

32 +

33 + messages = response_previous_input_to_harmony(chat_msg)

34 +

35 + assert len(messages) == 1

36 + assert messages[0].content[0].text == ""

37 +

38 + def test_tool_message_with_string_content(self):

39 + """Test parsing tool message with string content."""

40 + chat_msg = {

41 + "role": "tool",

42 + "name": "get_weather",

43 + "content": "The weather in San Francisco is sunny, 72°F",

44 + }

45 +

46 + messages = response_previous_input_to_harmony(chat_msg)

47 +

48 + assert len(messages) == 1

49 + assert messages[0].author.role == Role.TOOL

50 + assert messages[0].author.name == "functions.get_weather"

51 + assert (

52 + messages[0].content[0].text == "The weather in San Francisco is sunny, 72°F"

53 + )

54 + assert messages[0].channel == "commentary"

55 +

56 + def test_tool_message_with_array_content(self):

57 + """Test parsing tool message with array content."""

58 + chat_msg = {

59 + "role": "tool",

60 + "name": "search_results",

61 + "content": [

62 + {"type": "text", "text": "Result 1: "},

63 + {"type": "text", "text": "Result 2: "},

64 + {

65 + "type": "image",

66 + "url": "http://example.com/img.png",

67 + }, # Should be ignored

68 + {"type": "text", "text": "Result 3"},

69 + ],

70 + }

71 +

72 + messages = response_previous_input_to_harmony(chat_msg)

73 +

74 + assert len(messages) == 1

75 + assert messages[0].author.role == Role.TOOL

76 + assert messages[0].author.name == "functions.search_results"

77 + assert messages[0].content[0].text == "Result 1: Result 2: Result 3"

78 +

79 + def test_tool_message_with_empty_content(self):

80 + """Test parsing tool message with None content."""

81 + chat_msg = {

82 + "role": "tool",

83 + "name": "empty_tool",

84 + "content": None,

85 + }

86 +

87 + messages = response_previous_input_to_harmony(chat_msg)

88 +

89 + assert len(messages) == 1

90 + assert messages[0].author.role == Role.TOOL

91 + assert messages[0].author.name == "functions.empty_tool"

92 + assert messages[0].content[0].text == ""

93 +

94 +

95 +class TestHarmonyToResponseOutput:

96 + """Tests for harmony_to_response_output function."""

97 +

98 + def test_commentary_with_no_recipient_creates_message(self):

99 + """Test that commentary with recipient=None (preambles) creates message items.

100 +

101 + Per Harmony format, preambles are intended to be shown to end-users,

102 + unlike analysis channel content which is hidden reasoning.

103 + See: https://cookbook.openai.com/articles/openai-harmony

104 + """

105 + message = Message.from_role_and_content(

106 + Role.ASSISTANT, "I will now search for the weather information."

107 + )

108 + message = message.with_channel("commentary")

109 + # recipient is None by default, representing a preamble

110 +

111 + output_items = harmony_to_response_output(message)

112 +

113 + assert len(output_items) == 1

114 + assert isinstance(output_items[0], ResponseOutputMessage)

115 + assert output_items[0].type == "message"

116 + assert output_items[0].role == "assistant"

117 + assert output_items[0].status == "completed"

118 + assert len(output_items[0].content) == 1

119 + assert output_items[0].content[0].type == "output_text"

120 + assert (

121 + output_items[0].content[0].text

122 + == "I will now search for the weather information."

123 + )

124 +

125 + def test_commentary_with_function_recipient_creates_function_call(self):

126 + """Test commentary with recipient='functions.X' creates function calls."""

127 + message = Message.from_role_and_content(

128 + Role.ASSISTANT, '{"location": "San Francisco", "units": "celsius"}'

129 + )

130 + message = message.with_channel("commentary")

131 + message = message.with_recipient("functions.get_weather")

132 +

133 + output_items = harmony_to_response_output(message)

134 +

135 + assert len(output_items) == 1

136 + assert isinstance(output_items[0], ResponseFunctionToolCall)

137 + assert output_items[0].type == "function_call"

138 + assert output_items[0].name == "get_weather"

139 + assert (

140 + output_items[0].arguments

141 + == '{"location": "San Francisco", "units": "celsius"}'

142 + )

143 + assert output_items[0].call_id.startswith("call_")

144 + assert output_items[0].id.startswith("fc_")

145 +

146 + def test_commentary_with_python_recipient_creates_reasoning(self):

147 + """Test that commentary with recipient='python' creates reasoning items."""

148 + message = Message.from_role_and_content(

149 + Role.ASSISTANT, "import numpy as np\nprint(np.array([1, 2, 3]))"

150 + )

151 + message = message.with_channel("commentary")

152 + message = message.with_recipient("python")

153 +

154 + output_items = harmony_to_response_output(message)

155 +

156 + assert len(output_items) == 1

157 + assert isinstance(output_items[0], ResponseReasoningItem)

158 + assert output_items[0].type == "reasoning"

159 + assert (

160 + output_items[0].content[0].text

161 + == "import numpy as np\nprint(np.array([1, 2, 3]))"

162 + )

163 +

164 + def test_commentary_with_browser_recipient_creates_reasoning(self):

165 + """Test that commentary with recipient='browser' creates reasoning items."""

166 + message = Message.from_role_and_content(

167 + Role.ASSISTANT, "Navigating to the specified URL"

168 + )

169 + message = message.with_channel("commentary")

170 + message = message.with_recipient("browser")

171 +

172 + output_items = harmony_to_response_output(message)

173 +

174 + assert len(output_items) == 1

175 + assert isinstance(output_items[0], ResponseReasoningItem)

176 + assert output_items[0].type == "reasoning"

177 + assert output_items[0].content[0].text == "Navigating to the specified URL"

178 +

179 + def test_commentary_with_container_recipient_creates_reasoning(self):

180 + """Test that commentary with recipient='container' creates reasoning items."""

181 + message = Message.from_role_and_content(

182 + Role.ASSISTANT, "Running command in container"

183 + )

184 + message = message.with_channel("commentary")

185 + message = message.with_recipient("container")

186 +

187 + output_items = harmony_to_response_output(message)

188 +

189 + assert len(output_items) == 1

190 + assert isinstance(output_items[0], ResponseReasoningItem)

191 + assert output_items[0].type == "reasoning"

192 + assert output_items[0].content[0].text == "Running command in container"

193 +

194 + def test_commentary_with_empty_content_and_no_recipient(self):

195 + """Test edge case: empty commentary with recipient=None."""

196 + message = Message.from_role_and_content(Role.ASSISTANT, "")

197 + message = message.with_channel("commentary")

198 +

199 + output_items = harmony_to_response_output(message)

200 +

201 + assert len(output_items) == 1

202 + assert isinstance(output_items[0], ResponseOutputMessage)

203 + assert output_items[0].content[0].text == ""

204 +

205 + def test_commentary_with_multiple_contents_and_no_recipient(self):

206 + """Test multiple content items in commentary with no recipient."""

207 + contents = [

208 + TextContent(text="Step 1: Analyze the request"),

209 + TextContent(text="Step 2: Prepare to call functions"),

210 + ]

211 + message = Message.from_role_and_contents(Role.ASSISTANT, contents)

212 + message = message.with_channel("commentary")

213 +

214 + output_items = harmony_to_response_output(message)

215 +

216 + # _parse_final_message returns single ResponseOutputMessage with

217 + # multiple contents

218 + assert len(output_items) == 1

219 + assert isinstance(output_items[0], ResponseOutputMessage)

220 + assert len(output_items[0].content) == 2

221 + assert output_items[0].content[0].text == "Step 1: Analyze the request"

222 + assert output_items[0].content[1].text == "Step 2: Prepare to call functions"

223 +

224 + def test_commentary_with_multiple_function_calls(self):

225 + """Test multiple function calls in commentary channel."""

226 + contents = [

227 + TextContent(text='{"location": "San Francisco"}'),

228 + TextContent(text='{"location": "New York"}'),

229 + ]

230 + message = Message.from_role_and_contents(Role.ASSISTANT, contents)

231 + message = message.with_channel("commentary")

232 + message = message.with_recipient("functions.get_weather")

233 +

234 + output_items = harmony_to_response_output(message)

235 +

236 + assert len(output_items) == 2

237 + assert all(isinstance(item, ResponseFunctionToolCall) for item in output_items)

238 + assert output_items[0].name == "get_weather"

239 + assert output_items[1].name == "get_weather"

240 + assert output_items[0].arguments == '{"location": "San Francisco"}'

241 + assert output_items[1].arguments == '{"location": "New York"}'

242 +

243 + def test_commentary_with_unknown_recipient_creates_mcp_call(self):

244 + """Test that commentary with unknown recipient creates MCP call."""

245 + message = Message.from_role_and_content(Role.ASSISTANT, '{"arg": "value"}')

246 + message = message.with_channel("commentary")

247 + message = message.with_recipient("custom_tool")

248 +

249 + output_items = harmony_to_response_output(message)

250 +

251 + assert len(output_items) == 1

252 + assert isinstance(output_items[0], McpCall)

253 + assert output_items[0].type == "mcp_call"

254 + assert output_items[0].name == "custom_tool"

255 + assert output_items[0].server_label == "custom_tool"

256 +

257 + def test_analysis_channel_creates_reasoning(self):

258 + """Test that analysis channel creates reasoning items."""

259 + message = Message.from_role_and_content(

260 + Role.ASSISTANT, "Analyzing the problem step by step..."

261 + )

262 + message = message.with_channel("analysis")

263 +

264 + output_items = harmony_to_response_output(message)

265 +

266 + assert len(output_items) == 1

267 + assert isinstance(output_items[0], ResponseReasoningItem)

268 + assert output_items[0].type == "reasoning"

269 + assert (

270 + output_items[0].content[0].text == "Analyzing the problem step by step..."

271 + )

272 +

273 + def test_non_assistant_message_returns_empty(self):

274 + """Test that non-assistant messages return empty list.

275 +

276 + Per the implementation, tool messages to assistant (e.g., search results)

277 + are not included in final output to align with OpenAI behavior.

278 + """

279 + message = Message.from_author_and_content(

280 + Author.new(Role.TOOL, "functions.get_weather"),

281 + "The weather is sunny, 72°F",

282 + )

283 +

284 + output_items = harmony_to_response_output(message)

285 +

286 + assert len(output_items) == 0

287 +

288 +

289 +def test_parse_mcp_call_basic() -> None:

290 + """Test that MCP calls are parsed with correct type and server_label."""

291 + message = Message.from_role_and_content(Role.ASSISTANT, '{"path": "/tmp"}')

292 + message = message.with_recipient("filesystem")

293 + message = message.with_channel("commentary")

294 +

295 + output_items = harmony_to_response_output(message)

296 +

297 + assert len(output_items) == 1

298 + assert isinstance(output_items[0], McpCall)

299 + assert output_items[0].type == "mcp_call"

300 + assert output_items[0].name == "filesystem"

301 + assert output_items[0].server_label == "filesystem"

302 + assert output_items[0].arguments == '{"path": "/tmp"}'

303 + assert output_items[0].status == "completed"

304 +

305 +

306 +def test_parse_mcp_call_dotted_recipient() -> None:

307 + """Test that dotted recipients extract the tool name correctly."""

308 + message = Message.from_role_and_content(Role.ASSISTANT, '{"cmd": "ls"}')

309 + message = message.with_recipient("repo_browser.list")

310 + message = message.with_channel("commentary")

311 +

312 + output_items = harmony_to_response_output(message)

313 +

314 + assert len(output_items) == 1

315 + assert isinstance(output_items[0], McpCall)

316 + assert output_items[0].name == "list"

317 + assert output_items[0].server_label == "repo_browser"

318 +

319 +

320 +def test_mcp_vs_function_call() -> None:

321 + """Test that function calls are not parsed as MCP calls."""

322 + func_message = Message.from_role_and_content(Role.ASSISTANT, '{"arg": "value"}')

323 + func_message = func_message.with_recipient("functions.my_tool")

324 + func_message = func_message.with_channel("commentary")

325 +

326 + func_items = harmony_to_response_output(func_message)

327 +

328 + assert len(func_items) == 1

329 + assert not isinstance(func_items[0], McpCall)

330 + assert func_items[0].type == "function_call"

331 +

332 +

333 +def test_mcp_vs_builtin_tools() -> None:

334 + """Test that built-in tools (python, container) are not parsed as MCP calls."""

335 + # Test python (built-in tool) - should be reasoning, not MCP

336 + python_message = Message.from_role_and_content(Role.ASSISTANT, "print('hello')")

337 + python_message = python_message.with_recipient("python")

338 + python_message = python_message.with_channel("commentary")

339 +

340 + python_items = harmony_to_response_output(python_message)

341 +

342 + assert len(python_items) == 1

343 + assert not isinstance(python_items[0], McpCall)

344 + assert python_items[0].type == "reasoning"

345 +

346 +

347 +def test_parser_state_to_response_output_commentary_channel() -> None:

348 + """Test parser_state_to_response_output with commentary

349 + channel and various recipients."""

350 + from unittest.mock import Mock

351 +

352 + # Test 1: functions.* recipient -> should return function tool call

353 + parser_func = Mock()

354 + parser_func.current_content = '{"arg": "value"}'

355 + parser_func.current_role = Role.ASSISTANT

356 + parser_func.current_channel = "commentary"

357 + parser_func.current_recipient = "functions.my_tool"

358 +

359 + func_items = parser_state_to_response_output(parser_func)

360 +

361 + assert len(func_items) == 1

362 + assert not isinstance(func_items[0], McpCall)

363 + assert func_items[0].type == "function_call"

364 + assert func_items[0].name == "my_tool"

365 + assert func_items[0].status == "in_progress"

366 +

367 + # Test 2: MCP tool (not builtin) -> should return MCP call

368 + parser_mcp = Mock()

369 + parser_mcp.current_content = '{"path": "/tmp"}'

370 + parser_mcp.current_role = Role.ASSISTANT

371 + parser_mcp.current_channel = "commentary"

372 + parser_mcp.current_recipient = "filesystem"

373 +

374 + mcp_items = parser_state_to_response_output(parser_mcp)

375 +

376 + assert len(mcp_items) == 1

377 + assert isinstance(mcp_items[0], McpCall)

378 + assert mcp_items[0].type == "mcp_call"

379 + assert mcp_items[0].name == "filesystem"

380 + assert mcp_items[0].server_label == "filesystem"

381 + assert mcp_items[0].status == "in_progress"

382 +

383 + # Test 3: Built-in tool (python)

384 + # should NOT return MCP call, returns reasoning (internal tool interaction)

385 + parser_builtin = Mock()

386 + parser_builtin.current_content = "print('hello')"

387 + parser_builtin.current_role = Role.ASSISTANT

388 + parser_builtin.current_channel = "commentary"

389 + parser_builtin.current_recipient = "python"

390 +

391 + builtin_items = parser_state_to_response_output(parser_builtin)

392 +

393 + # Built-in tools explicitly return reasoning

394 + assert len(builtin_items) == 1

395 + assert not isinstance(builtin_items[0], McpCall)

396 + assert builtin_items[0].type == "reasoning"

397 +

398 + # Test 4: No recipient (preamble) → should return message, not reasoning

399 + parser_preamble = Mock()

400 + parser_preamble.current_content = "I'll search for that information now."

401 + parser_preamble.current_role = Role.ASSISTANT

402 + parser_preamble.current_channel = "commentary"

403 + parser_preamble.current_recipient = None

404 +

405 + preamble_items = parser_state_to_response_output(parser_preamble)

406 +

407 + assert len(preamble_items) == 1

408 + assert isinstance(preamble_items[0], ResponseOutputMessage)

409 + assert preamble_items[0].type == "message"

410 + assert preamble_items[0].content[0].text == "I'll search for that information now."

411 + assert preamble_items[0].status == "incomplete" # streaming

412 +

413 +

414 +def test_parser_state_to_response_output_analysis_channel() -> None:

415 + """Test parser_state_to_response_output with analysis

416 + channel and various recipients."""

417 + from unittest.mock import Mock

418 +

419 + # Test 1: functions.* recipient -> should return function tool call

420 + parser_func = Mock()

421 + parser_func.current_content = '{"arg": "value"}'

422 + parser_func.current_role = Role.ASSISTANT

423 + parser_func.current_channel = "analysis"

424 + parser_func.current_recipient = "functions.my_tool"

425 +

426 + func_items = parser_state_to_response_output(parser_func)

427 +

428 + assert len(func_items) == 1

429 + assert not isinstance(func_items[0], McpCall)

430 + assert func_items[0].type == "function_call"

431 + assert func_items[0].name == "my_tool"

432 + assert func_items[0].status == "in_progress"

433 +

434 + # Test 2: MCP tool (not builtin) -> should return MCP call

435 + parser_mcp = Mock()

436 + parser_mcp.current_content = '{"query": "test"}'

437 + parser_mcp.current_role = Role.ASSISTANT

438 + parser_mcp.current_channel = "analysis"

439 + parser_mcp.current_recipient = "database"

440 +

441 + mcp_items = parser_state_to_response_output(parser_mcp)

442 +

443 + assert len(mcp_items) == 1

444 + assert isinstance(mcp_items[0], McpCall)

445 + assert mcp_items[0].type == "mcp_call"

446 + assert mcp_items[0].name == "database"

447 + assert mcp_items[0].server_label == "database"

448 + assert mcp_items[0].status == "in_progress"

449 +

450 + # Test 3: Built-in tool (container)

451 + # should NOT return MCP call, falls through to reasoning

452 + parser_builtin = Mock()

453 + parser_builtin.current_content = "docker run"

454 + parser_builtin.current_role = Role.ASSISTANT

455 + parser_builtin.current_channel = "analysis"

456 + parser_builtin.current_recipient = "container"

457 +

458 + builtin_items = parser_state_to_response_output(parser_builtin)

459 +

460 + # Should fall through to reasoning logic

461 + assert len(builtin_items) == 1

462 + assert not isinstance(builtin_items[0], McpCall)

463 + assert builtin_items[0].type == "reasoning"

tests/entrypoints/openai/responses/test_mcp_tools.py

+5 -5

@@ -97,16 +97,16 @@ class TestMCPToolServerUnit:

97 assert server.get_tool_description("test_server", allowed_tools=[]) is None

99 def test_builtin_tools_consistency(self):

100 - """MCP_BUILTIN_TOOLS must match _BUILTIN_TOOL_TO_MCP_SERVER_LABEL values."""

100 + """MCP_BUILTIN_TOOLS must match BUILTIN_TOOL_TO_MCP_SERVER_LABEL values."""

101 from vllm.entrypoints.openai.parser.harmony_utils import (

102 - _BUILTIN_TOOL_TO_MCP_SERVER_LABEL,

102 + BUILTIN_TOOL_TO_MCP_SERVER_LABEL,

103 MCP_BUILTIN_TOOLS,

104 )

105

106 - assert set(_BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values()) == MCP_BUILTIN_TOOLS, (

106 + assert set(BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values()) == MCP_BUILTIN_TOOLS, (

107 f"MCP_BUILTIN_TOOLS {MCP_BUILTIN_TOOLS} does not match "

108 - f"_BUILTIN_TOOL_TO_MCP_SERVER_LABEL values "

109 - f"{set(_BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values())}"

108 + f"BUILTIN_TOOL_TO_MCP_SERVER_LABEL values "

109 + f"{set(BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values())}"

110 )

111

112

vllm/entrypoints/openai/parser/harmony_utils.py

+2 -516

@@ -2,27 +2,9 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 import datetime

5 -import json

5 from collections.abc import Iterable, Sequence

6 from typing import Literal

9 -from openai.types.responses import (

10 - ResponseFunctionToolCall,

11 - ResponseOutputItem,

12 - ResponseOutputMessage,

13 - ResponseOutputText,

14 - ResponseReasoningItem,

15 -)

16 -from openai.types.responses.response_function_web_search import (

17 - ActionFind,

18 - ActionOpenPage,

19 - ActionSearch,

20 - ResponseFunctionWebSearch,

21 -)

22 -from openai.types.responses.response_output_item import McpCall

23 -from openai.types.responses.response_reasoning_item import (

24 - Content as ResponseReasoningTextContent,

25 -)

8 from openai.types.responses.tool import Tool

9 from openai_harmony import (

10 Author,

@@ -38,17 +20,10 @@ from openai_harmony import (

20 ToolDescription,

21 load_harmony_encoding,

22 )

41 -from openai_harmony import Message as OpenAIHarmonyMessage

42 -from openai_harmony import Role as OpenAIHarmonyRole

24 from vllm import envs

25 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionToolsParam

46 -from vllm.entrypoints.openai.responses.protocol import (

47 - ResponseInputOutputItem,

48 - ResponsesRequest,

49 -)

26 from vllm.logger import init_logger

51 -from vllm.utils import random_uuid

28 logger = init_logger(__name__)

@@ -64,14 +39,14 @@ _harmony_encoding = None

39 # they are available and requested by the user.

40 # Tool args are provided by MCP tool descriptions. Output

41 # of the tools are stringified.

67 -_BUILTIN_TOOL_TO_MCP_SERVER_LABEL: dict[str, str] = {

42 +BUILTIN_TOOL_TO_MCP_SERVER_LABEL: dict[str, str] = {

43 "python": "code_interpreter",

44 "browser": "web_search_preview",

45 "container": "container",

46 }

48 # Derive MCP_BUILTIN_TOOLS from the canonical mapping

74 -MCP_BUILTIN_TOOLS: set[str] = set(_BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values())

49 +MCP_BUILTIN_TOOLS: set[str] = set(BUILTIN_TOOL_TO_MCP_SERVER_LABEL.values())

52 def has_custom_tools(tool_types: set[str]) -> bool:

@@ -179,55 +154,6 @@ def get_user_message(content: str) -> Message:

154 return Message.from_role_and_content(Role.USER, content)

155

156

182 -def parse_response_input(

183 - response_msg: ResponseInputOutputItem,

184 - prev_responses: list[ResponseOutputItem | ResponseReasoningItem],

185 -) -> Message:

186 - if not isinstance(response_msg, dict):

187 - response_msg = response_msg.model_dump()

188 - if "type" not in response_msg or response_msg["type"] == "message":

189 - role = response_msg["role"]

190 - content = response_msg["content"]

191 - # Add prefix for developer messages.

192 - # <|start|>developer<|message|># Instructions {instructions}<|end|>

193 - text_prefix = "Instructions:\n" if role == "developer" else ""

194 - if isinstance(content, str):

195 - msg = Message.from_role_and_content(role, text_prefix + content)

196 - else:

197 - contents = [TextContent(text=text_prefix + c["text"]) for c in content]

198 - msg = Message.from_role_and_contents(role, contents)

199 - if role == "assistant":

200 - msg = msg.with_channel("final")

201 - elif response_msg["type"] == "function_call_output":

202 - call_id = response_msg["call_id"]

203 - call_response: ResponseFunctionToolCall | None = None

204 - for prev_response in reversed(prev_responses):

205 - if (

206 - isinstance(prev_response, ResponseFunctionToolCall)

207 - and prev_response.call_id == call_id

208 - ):

209 - call_response = prev_response

210 - break

211 - if call_response is None:

212 - raise ValueError(f"No call message found for {call_id}")

213 - msg = Message.from_author_and_content(

214 - Author.new(Role.TOOL, f"functions.{call_response.name}"),

215 - response_msg["output"],

216 - )

217 - elif response_msg["type"] == "reasoning":

218 - content = response_msg["content"]

219 - assert len(content) == 1

220 - msg = Message.from_role_and_content(Role.ASSISTANT, content[0]["text"])

221 - elif response_msg["type"] == "function_call":

222 - msg = Message.from_role_and_content(Role.ASSISTANT, response_msg["arguments"])

223 - msg = msg.with_channel("commentary")

224 - msg = msg.with_recipient(f"functions.{response_msg['name']}")

225 - msg = msg.with_content_type("json")

226 - else:

227 - raise ValueError(f"Unknown input type: {response_msg['type']}")

228 - return msg

229 -

230 -

157 def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]:

158 """

159 Parse a list of messages from request.messages in the Chat Completion API to

@@ -390,139 +316,6 @@ def parse_chat_input_to_harmony_message(

316 return msgs

317

318

393 -def parse_input_to_harmony_message(chat_msg) -> list[Message]:

394 - """Parse a message from request.previous_input_messages

395 - into Harmony messages.

396 -

397 - Supports both OpenAI chat format ({"role": "..."}) and

398 - Harmony format ({"author": {"role": "..."}}).

399 - """

400 - if not isinstance(chat_msg, dict):

401 - chat_msg = chat_msg.model_dump(exclude_none=True)

402 -

403 - if "author" in chat_msg and isinstance(chat_msg.get("author"), dict):

404 - return [_parse_harmony_format_message(chat_msg)]

405 -

406 - return _parse_chat_format_message(chat_msg)

407 -

408 -

409 -def _parse_harmony_format_message(chat_msg: dict) -> Message:

410 - """Reconstruct a Message from Harmony-format dict,

411 - preserving channel, recipient, and content_type."""

412 - author_dict = chat_msg["author"]

413 - role = author_dict.get("role")

414 - name = author_dict.get("name")

415 -

416 - raw_content = chat_msg.get("content", "")

417 - if isinstance(raw_content, list):

418 - # TODO: Support refusal and non-text content types.

419 - contents = [TextContent(text=c.get("text", "")) for c in raw_content]

420 - elif isinstance(raw_content, str):

421 - contents = [TextContent(text=raw_content)]

422 - else:

423 - contents = [TextContent(text="")]

424 -

425 - if name:

426 - msg = Message.from_author_and_contents(Author.new(Role(role), name), contents)

427 - else:

428 - msg = Message.from_role_and_contents(Role(role), contents)

429 -

430 - channel = chat_msg.get("channel")

431 - if channel:

432 - msg = msg.with_channel(channel)

433 - recipient = chat_msg.get("recipient")

434 - if recipient:

435 - msg = msg.with_recipient(recipient)

436 - content_type = chat_msg.get("content_type")

437 - if content_type:

438 - msg = msg.with_content_type(content_type)

439 -

440 - return msg

441 -

442 -

443 -def _parse_chat_format_message(chat_msg: dict) -> list[Message]:

444 - """Parse an OpenAI chat-format dict into Harmony messages."""

445 - role = chat_msg.get("role")

446 - if role is None:

447 - raise ValueError(f"Message has no 'role' key: {chat_msg}")

448 -

449 - # Assistant message with tool calls

450 - tool_calls = chat_msg.get("tool_calls")

451 - if role == "assistant" and tool_calls:

452 - msgs: list[Message] = []

453 - for call in tool_calls:

454 - func = call.get("function", {})

455 - name = func.get("name", "")

456 - arguments = func.get("arguments", "") or ""

457 - msg = Message.from_role_and_content(Role.ASSISTANT, arguments)

458 - msg = msg.with_channel("commentary")

459 - msg = msg.with_recipient(f"functions.{name}")

460 - msg = msg.with_content_type("json")

461 - msgs.append(msg)

462 - return msgs

463 -

464 - # Tool role message (tool output)

465 - if role == "tool":

466 - name = chat_msg.get("name", "")

467 - if name and not name.startswith("functions."):

468 - name = f"functions.{name}"

469 - content = chat_msg.get("content", "") or ""

470 - content = flatten_chat_text_content(content)

471 - # NOTE: .with_recipient("assistant") is required on tool messages

472 - # to match parse_chat_input_to_harmony_message behavior and ensure

473 - # proper routing in the Harmony protocol.

474 - msg = (

475 - Message.from_author_and_content(Author.new(Role.TOOL, name), content)

476 - .with_channel("commentary")

477 - .with_recipient("assistant")

478 - )

479 - return [msg]

480 -

481 - # Default: user/assistant/system messages

482 - content = chat_msg.get("content", "")

483 - if isinstance(content, str):

484 - contents = [TextContent(text=content)]

485 - else:

486 - # TODO: Support refusal.

487 - contents = [TextContent(text=c.get("text", "")) for c in content]

488 - msg = Message.from_role_and_contents(role, contents)

489 - return [msg]

490 -

491 -

492 -def construct_harmony_previous_input_messages(

493 - request: ResponsesRequest,

494 -) -> list[OpenAIHarmonyMessage]:

495 - messages: list[OpenAIHarmonyMessage] = []

496 - if request.previous_input_messages:

497 - for message in request.previous_input_messages:

498 - # Handle both OpenAIHarmonyMessage objects and dictionary inputs

499 - if isinstance(message, OpenAIHarmonyMessage):

500 - message_role = message.author.role

501 - # To match OpenAI, instructions, reasoning and tools are

502 - # always taken from the most recent Responses API request

503 - # not carried over from previous requests

504 - if (

505 - message_role == OpenAIHarmonyRole.SYSTEM

506 - or message_role == OpenAIHarmonyRole.DEVELOPER

507 - ):

508 - continue

509 - messages.append(message)

510 - else:

511 - harmony_messages = parse_input_to_harmony_message(message)

512 - for harmony_msg in harmony_messages:

513 - message_role = harmony_msg.author.role

514 - # To match OpenAI, instructions, reasoning and tools are

515 - # always taken from the most recent Responses API request

516 - # not carried over from previous requests

517 - if (

518 - message_role == OpenAIHarmonyRole.SYSTEM

519 - or message_role == OpenAIHarmonyRole.DEVELOPER

520 - ):

521 - continue

522 - messages.append(harmony_msg)

523 - return messages

524 -

525 -

319 def render_for_completion(messages: list[Message]) -> list[int]:

320 conversation = Conversation.from_messages(messages)

321 token_ids = get_encoding().render_conversation_for_completion(

@@ -531,313 +324,6 @@ def render_for_completion(messages: list[Message]) -> list[int]:

324 return token_ids

325

326

534 -def _parse_browser_tool_call(message: Message, recipient: str) -> ResponseOutputItem:

535 - """Parse browser tool calls (search, open, find) into web search items."""

536 - if len(message.content) != 1:

537 - raise ValueError("Invalid number of contents in browser message")

538 - content = message.content[0]

539 -

540 - # Parse JSON args (with retry detection)

541 - try:

542 - browser_call = json.loads(content.text)

543 - except json.JSONDecodeError:

544 - logger.warning(

545 - "Invalid JSON in browser tool call, using error placeholder: %s",

546 - content.text,

547 - )

548 - json_retry_output_message = (

549 - f"Invalid JSON args, caught and retried: {content.text}"

550 - )

551 - browser_call = {

552 - "query": json_retry_output_message,

553 - "url": json_retry_output_message,

554 - "pattern": json_retry_output_message,

555 - }

556 -

557 - # Create appropriate action based on recipient

558 - if recipient == "browser.search":

559 - action = ActionSearch(

560 - query=f"cursor:{browser_call.get('query', '')}", type="search"

561 - )

562 - elif recipient == "browser.open":

563 - action = ActionOpenPage(

564 - url=f"cursor:{browser_call.get('url', '')}", type="open_page"

565 - )

566 - elif recipient == "browser.find":

567 - action = ActionFind(

568 - pattern=browser_call.get("pattern", ""),

569 - url=f"cursor:{browser_call.get('url', '')}",

570 - type="find",

571 - )

572 - else:

573 - raise ValueError(f"Unknown browser action: {recipient}")

574 -

575 - return ResponseFunctionWebSearch(

576 - id=f"ws_{random_uuid()}",

577 - action=action,

578 - status="completed",

579 - type="web_search_call",

580 - )

581 -

582 -

583 -def _parse_function_call(message: Message, recipient: str) -> list[ResponseOutputItem]:

584 - """Parse function calls into function tool call items."""

585 - function_name = recipient.split(".")[-1]

586 - output_items = []

587 - for content in message.content:

588 - random_id = random_uuid()

589 - response_item = ResponseFunctionToolCall(

590 - arguments=content.text,

591 - call_id=f"call_{random_id}",

592 - type="function_call",

593 - name=function_name,

594 - id=f"fc_{random_id}",

595 - )

596 - output_items.append(response_item)

597 - return output_items

598 -

599 -

600 -def _parse_reasoning(message: Message) -> list[ResponseOutputItem]:

601 - """Parse reasoning/analysis content into reasoning items."""

602 - output_items = []

603 - for content in message.content:

604 - reasoning_item = ResponseReasoningItem(

605 - id=f"rs_{random_uuid()}",

606 - summary=[],

607 - type="reasoning",

608 - content=[

609 - ResponseReasoningTextContent(text=content.text, type="reasoning_text")

610 - ],

611 - status=None,

612 - )

613 - output_items.append(reasoning_item)

614 - return output_items

615 -

616 -

617 -def _parse_final_message(message: Message) -> ResponseOutputItem:

618 - """Parse final channel messages into output message items."""

619 - contents = []

620 - for content in message.content:

621 - output_text = ResponseOutputText(

622 - text=content.text,

623 - annotations=[], # TODO

624 - type="output_text",

625 - logprobs=None, # TODO

626 - )

627 - contents.append(output_text)

628 - return ResponseOutputMessage(

629 - id=f"msg_{random_uuid()}",

630 - content=contents,

631 - role=message.author.role,

632 - status="completed",

633 - type="message",

634 - )

635 -

636 -

637 -def _parse_mcp_recipient(recipient: str) -> tuple[str, str]:

638 - """

639 - Parse MCP recipient into (server_label, tool_name).

640 -

641 - For dotted recipients like "repo_browser.list":

642 - - server_label: "repo_browser" (namespace/server)

643 - - tool_name: "list" (specific tool)

644 -

645 - For simple recipients like "filesystem":

646 - - server_label: "filesystem"

647 - - tool_name: "filesystem"

648 - """

649 - if "." in recipient:

650 - server_label = recipient.split(".")[0]

651 - tool_name = recipient.split(".")[-1]

652 - else:

653 - server_label = recipient

654 - tool_name = recipient

655 - return server_label, tool_name

656 -

657 -

658 -def _parse_mcp_call(message: Message, recipient: str) -> list[ResponseOutputItem]:

659 - """Parse MCP calls into MCP call items."""

660 - # Handle built-in tools that need server_label mapping

661 - if recipient in _BUILTIN_TOOL_TO_MCP_SERVER_LABEL:

662 - server_label = _BUILTIN_TOOL_TO_MCP_SERVER_LABEL[recipient]

663 - tool_name = recipient

664 - else:

665 - server_label, tool_name = _parse_mcp_recipient(recipient)

666 -

667 - output_items = []

668 - for content in message.content:

669 - response_item = McpCall(

670 - arguments=content.text,

671 - type="mcp_call",

672 - name=tool_name,

673 - server_label=server_label,

674 - id=f"mcp_{random_uuid()}",

675 - status="completed",

676 - )

677 - output_items.append(response_item)

678 - return output_items

679 -

680 -

681 -def _parse_message_no_recipient(

682 - message: Message,

683 -) -> list[ResponseOutputItem]:

684 - """Parse a Harmony message with no recipient based on its channel."""

685 - if message.channel == "analysis":

686 - return _parse_reasoning(message)

687 -

688 - if message.channel in ("commentary", "final"):

689 - # Per Harmony format, preambles (commentary with no recipient) and

690 - # final channel content are both intended to be shown to end-users.

691 - # See: https://cookbook.openai.com/articles/openai-harmony

692 - return [_parse_final_message(message)]

693 -

694 - raise ValueError(f"Unknown channel: {message.channel}")

695 -

696 -

697 -def parse_output_message(message: Message) -> list[ResponseOutputItem]:

698 - """

699 - Parse a Harmony message into a list of output response items.

700 - """

701 - if message.author.role != "assistant":

702 - # This is a message from a tool to the assistant (e.g., search result).

703 - # Don't include it in the final output for now. This aligns with

704 - # OpenAI's behavior on models like o4-mini.

705 - return []

706 -

707 - output_items: list[ResponseOutputItem] = []

708 - recipient = message.recipient

709 -

710 - if recipient is not None:

711 - # Browser tool calls (browser.search, browser.open, browser.find)

712 - if recipient.startswith("browser."):

713 - output_items.append(_parse_browser_tool_call(message, recipient))

714 -

715 - # Function calls (should only happen on commentary channel)

716 - elif message.channel == "commentary" and recipient.startswith("functions."):

717 - output_items.extend(_parse_function_call(message, recipient))

718 -

719 - # Built-in MCP tools (python, browser, container)

720 - elif recipient in _BUILTIN_TOOL_TO_MCP_SERVER_LABEL:

721 - output_items.extend(_parse_reasoning(message))

722 -

723 - # All other recipients are MCP calls

724 - else:

725 - output_items.extend(_parse_mcp_call(message, recipient))

726 -

727 - # No recipient - handle based on channel for non-tool messages

728 - else:

729 - output_items.extend(_parse_message_no_recipient(message))

730 -

731 - return output_items

732 -

733 -

734 -def parse_remaining_state(parser: StreamableParser) -> list[ResponseOutputItem]:

735 - if not parser.current_content:

736 - return []

737 - if parser.current_role != Role.ASSISTANT:

738 - return []

739 - current_recipient = parser.current_recipient

740 - if current_recipient is not None and current_recipient.startswith("browser."):

741 - return []

742 -

743 - if current_recipient and parser.current_channel in ("commentary", "analysis"):

744 - if current_recipient.startswith("functions."):

745 - rid = random_uuid()

746 - return [

747 - ResponseFunctionToolCall(

748 - arguments=parser.current_content,

749 - call_id=f"call_{rid}",

750 - type="function_call",

751 - name=current_recipient.split(".")[-1],

752 - id=f"fc_{rid}",

753 - status="in_progress",

754 - )

755 - ]

756 - # Built-in MCP tools (python, browser, container)

757 - elif current_recipient in _BUILTIN_TOOL_TO_MCP_SERVER_LABEL:

758 - return [

759 - ResponseReasoningItem(

760 - id=f"rs_{random_uuid()}",

761 - summary=[],

762 - type="reasoning",

763 - content=[

764 - ResponseReasoningTextContent(

765 - text=parser.current_content, type="reasoning_text"

766 - )

767 - ],

768 - status=None,

769 - )

770 - ]

771 - # All other recipients are MCP calls

772 - else:

773 - rid = random_uuid()

774 - server_label, tool_name = _parse_mcp_recipient(current_recipient)

775 - return [

776 - McpCall(

777 - arguments=parser.current_content,

778 - type="mcp_call",

779 - name=tool_name,

780 - server_label=server_label,

781 - id=f"mcp_{rid}",

782 - status="in_progress",

783 - )

784 - ]

785 -

786 - if parser.current_channel == "commentary":

787 - # Per Harmony format, preambles (commentary with no recipient) are

788 - # intended to be shown to end-users, unlike analysis channel content.

789 - output_text = ResponseOutputText(

790 - text=parser.current_content,

791 - annotations=[],

792 - type="output_text",

793 - logprobs=None,

794 - )

795 - return [

796 - ResponseOutputMessage(

797 - id=f"msg_{random_uuid()}",

798 - content=[output_text],

799 - role="assistant",

800 - status="incomplete",

801 - type="message",

802 - )

803 - ]

804 -

805 - if parser.current_channel == "analysis":

806 - return [

807 - ResponseReasoningItem(

808 - id=f"rs_{random_uuid()}",

809 - summary=[],

810 - type="reasoning",

811 - content=[

812 - ResponseReasoningTextContent(

813 - text=parser.current_content, type="reasoning_text"

814 - )

815 - ],

816 - status=None,

817 - )

818 - ]

819 -

820 - if parser.current_channel == "final":

821 - output_text = ResponseOutputText(

822 - text=parser.current_content,

823 - annotations=[], # TODO

824 - type="output_text",

825 - logprobs=None, # TODO

826 - )

827 - text_item = ResponseOutputMessage(

828 - id=f"msg_{random_uuid()}",

829 - content=[output_text],

830 - role="assistant",

831 - # if the parser still has messages (ie if the generator got cut

832 - # abruptly), this should be incomplete

833 - status="incomplete",

834 - type="message",

835 - )

836 - return [text_item]

837 -

838 - return []

839 -

840 -

327 def get_stop_tokens_for_assistant_actions() -> list[int]:

328 return get_encoding().stop_tokens_for_assistant_actions()

329

vllm/entrypoints/openai/responses/harmony.py new

+552

@@ -0,0 +1,552 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""

4 +Harmony ↔ Responses API conversion utilities.

5 +

6 +Handles two directions:

7 + 1. Response Input → Harmony Messages (input parsing)

8 + 2. Harmony Messages → Response Output Items (output parsing)

9 +"""

10 +

11 +import json

12 +

13 +from openai.types.responses import (

14 + ResponseFunctionToolCall,

15 + ResponseOutputItem,

16 + ResponseOutputMessage,

17 + ResponseOutputText,

18 + ResponseReasoningItem,

19 +)

20 +from openai.types.responses.response_function_web_search import (

21 + ActionFind,

22 + ActionOpenPage,

23 + ActionSearch,

24 + ResponseFunctionWebSearch,

25 +)

26 +from openai.types.responses.response_output_item import McpCall

27 +from openai.types.responses.response_reasoning_item import (

28 + Content as ResponseReasoningTextContent,

29 +)

30 +from openai_harmony import Author, Message, Role, StreamableParser, TextContent

31 +

32 +from vllm.entrypoints.openai.parser.harmony_utils import (

33 + BUILTIN_TOOL_TO_MCP_SERVER_LABEL,

34 + flatten_chat_text_content,

35 +)

36 +from vllm.entrypoints.openai.responses.protocol import (

37 + ResponseInputOutputItem,

38 + ResponsesRequest,

39 +)

40 +from vllm.logger import init_logger

41 +from vllm.utils import random_uuid

42 +

43 +logger = init_logger(__name__)

44 +

45 +# ---------------------------------------------------------------------------

46 +# 1. Private helpers for input parsing

47 +# ---------------------------------------------------------------------------

48 +

49 +

50 +def _parse_harmony_format_message(chat_msg: dict) -> Message:

51 + """Reconstruct a Message from Harmony-format dict,

52 + preserving channel, recipient, and content_type."""

53 + author_dict = chat_msg["author"]

54 + role = author_dict.get("role")

55 + name = author_dict.get("name")

56 +

57 + raw_content = chat_msg.get("content", "")

58 + if isinstance(raw_content, list):

59 + # TODO: Support refusal and non-text content types.

60 + contents = [TextContent(text=c.get("text", "")) for c in raw_content]

61 + elif isinstance(raw_content, str):

62 + contents = [TextContent(text=raw_content)]

63 + else:

64 + contents = [TextContent(text="")]

65 +

66 + if name:

67 + msg = Message.from_author_and_contents(Author.new(Role(role), name), contents)

68 + else:

69 + msg = Message.from_role_and_contents(Role(role), contents)

70 +

71 + channel = chat_msg.get("channel")

72 + if channel:

73 + msg = msg.with_channel(channel)

74 + recipient = chat_msg.get("recipient")

75 + if recipient:

76 + msg = msg.with_recipient(recipient)

77 + content_type = chat_msg.get("content_type")

78 + if content_type:

79 + msg = msg.with_content_type(content_type)

80 +

81 + return msg

82 +

83 +

84 +def _parse_chat_format_message(chat_msg: dict) -> list[Message]:

85 + """Parse an OpenAI chat-format dict into Harmony messages."""

86 + role = chat_msg.get("role")

87 + if role is None:

88 + raise ValueError(f"Message has no 'role' key: {chat_msg}")

89 +

90 + # Assistant message with tool calls

91 + tool_calls = chat_msg.get("tool_calls")

92 + if role == "assistant" and tool_calls:

93 + msgs: list[Message] = []

94 + for call in tool_calls:

95 + func = call.get("function", {})

96 + name = func.get("name", "")

97 + arguments = func.get("arguments", "") or ""

98 + msg = Message.from_role_and_content(Role.ASSISTANT, arguments)

99 + msg = msg.with_channel("commentary")

100 + msg = msg.with_recipient(f"functions.{name}")

101 + msg = msg.with_content_type("json")

102 + msgs.append(msg)

103 + return msgs

104 +

105 + # Tool role message (tool output)

106 + if role == "tool":

107 + name = chat_msg.get("name", "")

108 + if name and not name.startswith("functions."):

109 + name = f"functions.{name}"

110 + content = chat_msg.get("content", "") or ""

111 + content = flatten_chat_text_content(content)

112 + # NOTE: .with_recipient("assistant") is required on tool messages

113 + # to match parse_chat_input_to_harmony_message behavior and ensure

114 + # proper routing in the Harmony protocol.

115 + msg = (

116 + Message.from_author_and_content(Author.new(Role.TOOL, name), content)

117 + .with_channel("commentary")

118 + .with_recipient("assistant")

119 + )

120 + return [msg]

121 +

122 + # Default: user/assistant/system messages

123 + content = chat_msg.get("content", "")

124 + if isinstance(content, str):

125 + contents = [TextContent(text=content)]

126 + else:

127 + # TODO: Support refusal.

128 + contents = [TextContent(text=c.get("text", "")) for c in content]

129 + msg = Message.from_role_and_contents(role, contents)

130 + return [msg]

131 +

132 +

133 +# ---------------------------------------------------------------------------

134 +# 2. Public input parsing functions

135 +# ---------------------------------------------------------------------------

136 +

137 +

138 +def response_input_to_harmony(

139 + response_msg: ResponseInputOutputItem,

140 + prev_responses: list[ResponseOutputItem | ResponseReasoningItem],

141 +) -> Message:

142 + """Convert a single ResponseInputOutputItem into a Harmony Message."""

143 + if not isinstance(response_msg, dict):

144 + response_msg = response_msg.model_dump()

145 + if "type" not in response_msg or response_msg["type"] == "message":

146 + role = response_msg["role"]

147 + content = response_msg["content"]

148 + # Add prefix for developer messages.

149 + # <|start|>developer<|message|># Instructions {instructions}<|end|>

150 + text_prefix = "Instructions:\n" if role == "developer" else ""

151 + if isinstance(content, str):

152 + msg = Message.from_role_and_content(role, text_prefix + content)

153 + else:

154 + contents = [TextContent(text=text_prefix + c["text"]) for c in content]

155 + msg = Message.from_role_and_contents(role, contents)

156 + if role == "assistant":

157 + msg = msg.with_channel("final")

158 + elif response_msg["type"] == "function_call_output":

159 + call_id = response_msg["call_id"]

160 + call_response: ResponseFunctionToolCall | None = None

161 + for prev_response in reversed(prev_responses):

162 + if (

163 + isinstance(prev_response, ResponseFunctionToolCall)

164 + and prev_response.call_id == call_id

165 + ):

166 + call_response = prev_response

167 + break

168 + if call_response is None:

169 + raise ValueError(f"No call message found for {call_id}")

170 + msg = Message.from_author_and_content(

171 + Author.new(Role.TOOL, f"functions.{call_response.name}"),

172 + response_msg["output"],

173 + )

174 + elif response_msg["type"] == "reasoning":

175 + content = response_msg["content"]

176 + assert len(content) == 1

177 + msg = Message.from_role_and_content(Role.ASSISTANT, content[0]["text"])

178 + elif response_msg["type"] == "function_call":

179 + msg = Message.from_role_and_content(Role.ASSISTANT, response_msg["arguments"])

180 + msg = msg.with_channel("commentary")

181 + msg = msg.with_recipient(f"functions.{response_msg['name']}")

182 + msg = msg.with_content_type("json")

183 + else:

184 + raise ValueError(f"Unknown input type: {response_msg['type']}")

185 + return msg

186 +

187 +

188 +def response_previous_input_to_harmony(chat_msg) -> list[Message]:

189 + """Parse a message from request.previous_input_messages

190 + into Harmony messages.

191 +

192 + Supports both OpenAI chat format ({"role": "..."}) and

193 + Harmony format ({"author": {"role": "..."}}).

194 + """

195 + if not isinstance(chat_msg, dict):

196 + chat_msg = chat_msg.model_dump(exclude_none=True)

197 +

198 + if "author" in chat_msg and isinstance(chat_msg.get("author"), dict):

199 + return [_parse_harmony_format_message(chat_msg)]

200 +

201 + return _parse_chat_format_message(chat_msg)

202 +

203 +

204 +def construct_harmony_previous_input_messages(

205 + request: ResponsesRequest,

206 +) -> list[Message]:

207 + """Build a Harmony message list from request.previous_input_messages.

208 +

209 + Filters out system/developer messages to match OpenAI behavior where

210 + instructions are always taken from the most recent Responses API request.

211 + """

212 + messages: list[Message] = []

213 + if request.previous_input_messages:

214 + for message in request.previous_input_messages:

215 + # Handle both Message objects and dictionary inputs

216 + if isinstance(message, Message):

217 + message_role = message.author.role

218 + if message_role == Role.SYSTEM or message_role == Role.DEVELOPER:

219 + continue

220 + messages.append(message)

221 + else:

222 + harmony_messages = response_previous_input_to_harmony(message)

223 + for harmony_msg in harmony_messages:

224 + message_role = harmony_msg.author.role

225 + if message_role == Role.SYSTEM or message_role == Role.DEVELOPER:

226 + continue

227 + messages.append(harmony_msg)

228 + return messages

229 +

230 +

231 +# ---------------------------------------------------------------------------

232 +# 3. Private helpers for output parsing

233 +# ---------------------------------------------------------------------------

234 +

235 +

236 +def _parse_browser_tool_call(message: Message, recipient: str) -> ResponseOutputItem:

237 + """Parse browser tool calls (search, open, find) into web search items."""

238 + if len(message.content) != 1:

239 + raise ValueError("Invalid number of contents in browser message")

240 + content = message.content[0]

241 +

242 + # Parse JSON args (with retry detection)

243 + try:

244 + browser_call = json.loads(content.text)

245 + except json.JSONDecodeError:

246 + logger.warning(

247 + "Invalid JSON in browser tool call, using error placeholder: %s",

248 + content.text,

249 + )

250 + json_retry_output_message = (

251 + f"Invalid JSON args, caught and retried: {content.text}"

252 + )

253 + browser_call = {

254 + "query": json_retry_output_message,

255 + "url": json_retry_output_message,

256 + "pattern": json_retry_output_message,

257 + }

258 +

259 + # Create appropriate action based on recipient

260 + if recipient == "browser.search":

261 + action = ActionSearch(

262 + query=f"cursor:{browser_call.get('query', '')}", type="search"

263 + )

264 + elif recipient == "browser.open":

265 + action = ActionOpenPage(

266 + url=f"cursor:{browser_call.get('url', '')}", type="open_page"

267 + )

268 + elif recipient == "browser.find":

269 + action = ActionFind(

270 + pattern=browser_call.get("pattern", ""),

271 + url=f"cursor:{browser_call.get('url', '')}",

272 + type="find",

273 + )

274 + else:

275 + raise ValueError(f"Unknown browser action: {recipient}")

276 +

277 + return ResponseFunctionWebSearch(

278 + id=f"ws_{random_uuid()}",

279 + action=action,

280 + status="completed",

281 + type="web_search_call",

282 + )

283 +

284 +

285 +def _parse_function_call(message: Message, recipient: str) -> list[ResponseOutputItem]:

286 + """Parse function calls into function tool call items."""

287 + function_name = recipient.split(".")[-1]

288 + output_items = []

289 + for content in message.content:

290 + random_id = random_uuid()

291 + response_item = ResponseFunctionToolCall(

292 + arguments=content.text,

293 + call_id=f"call_{random_id}",

294 + type="function_call",

295 + name=function_name,

296 + id=f"fc_{random_id}",

297 + )

298 + output_items.append(response_item)

299 + return output_items

300 +

301 +

302 +def _parse_reasoning(message: Message) -> list[ResponseOutputItem]:

303 + """Parse reasoning/analysis content into reasoning items."""

304 + output_items = []

305 + for content in message.content:

306 + reasoning_item = ResponseReasoningItem(

307 + id=f"rs_{random_uuid()}",

308 + summary=[],

309 + type="reasoning",

310 + content=[

311 + ResponseReasoningTextContent(text=content.text, type="reasoning_text")

312 + ],

313 + status=None,

314 + )

315 + output_items.append(reasoning_item)

316 + return output_items

317 +

318 +

319 +def _parse_final_message(message: Message) -> ResponseOutputItem:

320 + """Parse final channel messages into output message items."""

321 + contents = []

322 + for content in message.content:

323 + output_text = ResponseOutputText(

324 + text=content.text,

325 + annotations=[], # TODO

326 + type="output_text",

327 + logprobs=None, # TODO

328 + )

329 + contents.append(output_text)

330 + return ResponseOutputMessage(

331 + id=f"msg_{random_uuid()}",

332 + content=contents,

333 + role=message.author.role,

334 + status="completed",

335 + type="message",

336 + )

337 +

338 +

339 +def _parse_mcp_recipient(recipient: str) -> tuple[str, str]:

340 + """Parse MCP recipient into (server_label, tool_name).

341 +

342 + For dotted recipients like "repo_browser.list":

343 + - server_label: "repo_browser" (namespace/server)

344 + - tool_name: "list" (specific tool)

345 +

346 + For simple recipients like "filesystem":

347 + - server_label: "filesystem"

348 + - tool_name: "filesystem"

349 + """

350 + if "." in recipient:

351 + server_label = recipient.split(".")[0]

352 + tool_name = recipient.split(".")[-1]

353 + else:

354 + server_label = recipient

355 + tool_name = recipient

356 + return server_label, tool_name

357 +

358 +

359 +def _parse_mcp_call(message: Message, recipient: str) -> list[ResponseOutputItem]:

360 + """Parse MCP calls into MCP call items."""

361 + # Handle built-in tools that need server_label mapping

362 + if recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:

363 + server_label = BUILTIN_TOOL_TO_MCP_SERVER_LABEL[recipient]

364 + tool_name = recipient

365 + else:

366 + server_label, tool_name = _parse_mcp_recipient(recipient)

367 +

368 + output_items = []

369 + for content in message.content:

370 + response_item = McpCall(

371 + arguments=content.text,

372 + type="mcp_call",

373 + name=tool_name,

374 + server_label=server_label,

375 + id=f"mcp_{random_uuid()}",

376 + status="completed",

377 + )

378 + output_items.append(response_item)

379 + return output_items

380 +

381 +

382 +def _parse_message_no_recipient(

383 + message: Message,

384 +) -> list[ResponseOutputItem]:

385 + """Parse a Harmony message with no recipient based on its channel."""

386 + if message.channel == "analysis":

387 + return _parse_reasoning(message)

388 +

389 + if message.channel in ("commentary", "final"):

390 + # Per Harmony format, preambles (commentary with no recipient) and

391 + # final channel content are both intended to be shown to end-users.

392 + # See: https://cookbook.openai.com/articles/openai-harmony

393 + return [_parse_final_message(message)]

394 +

395 + raise ValueError(f"Unknown channel: {message.channel}")

396 +

397 +

398 +# ---------------------------------------------------------------------------

399 +# 4. Public output parsing functions

400 +# ---------------------------------------------------------------------------

401 +

402 +

403 +def harmony_to_response_output(message: Message) -> list[ResponseOutputItem]:

404 + """Parse a Harmony message into a list of output response items.

405 +

406 + This is the main dispatcher that routes based on channel and recipient.

407 + """

408 + if message.author.role != "assistant":

409 + # This is a message from a tool to the assistant (e.g., search result).

410 + # Don't include it in the final output for now. This aligns with

411 + # OpenAI's behavior on models like o4-mini.

412 + return []

413 +

414 + output_items: list[ResponseOutputItem] = []

415 + recipient = message.recipient

416 +

417 + if recipient is not None:

418 + # Browser tool calls (browser.search, browser.open, browser.find)

419 + if recipient.startswith("browser."):

420 + output_items.append(_parse_browser_tool_call(message, recipient))

421 +

422 + # Function calls (should only happen on commentary channel)

423 + elif message.channel == "commentary" and recipient.startswith("functions."):

424 + output_items.extend(_parse_function_call(message, recipient))

425 +

426 + # Built-in MCP tools (python, browser, container)

427 + elif recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:

428 + output_items.extend(_parse_reasoning(message))

429 +

430 + # All other recipients are MCP calls

431 + else:

432 + output_items.extend(_parse_mcp_call(message, recipient))

433 +

434 + # No recipient - handle based on channel for non-tool messages

435 + else:

436 + output_items.extend(_parse_message_no_recipient(message))

437 +

438 + return output_items

439 +

440 +

441 +def parser_state_to_response_output(

442 + parser: StreamableParser,

443 +) -> list[ResponseOutputItem]:

444 + """Extract in-progress response items from incomplete parser state.

445 +

446 + Called when the parser has buffered content that hasn't formed a

447 + complete message yet (e.g., generation was cut short).

448 + """

449 + if not parser.current_content:

450 + return []

451 + if parser.current_role != Role.ASSISTANT:

452 + return []

453 + current_recipient = parser.current_recipient

454 + if current_recipient is not None and current_recipient.startswith("browser."):

455 + return []

456 +

457 + if current_recipient and parser.current_channel in ("commentary", "analysis"):

458 + if current_recipient.startswith("functions."):

459 + rid = random_uuid()

460 + return [

461 + ResponseFunctionToolCall(

462 + arguments=parser.current_content,

463 + call_id=f"call_{rid}",

464 + type="function_call",

465 + name=current_recipient.split(".")[-1],

466 + id=f"fc_{rid}",

467 + status="in_progress",

468 + )

469 + ]

470 + # Built-in MCP tools (python, browser, container)

471 + elif current_recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:

472 + return [

473 + ResponseReasoningItem(

474 + id=f"rs_{random_uuid()}",

475 + summary=[],

476 + type="reasoning",

477 + content=[

478 + ResponseReasoningTextContent(

479 + text=parser.current_content, type="reasoning_text"

480 + )

481 + ],

482 + status=None,

483 + )

484 + ]

485 + # All other recipients are MCP calls

486 + else:

487 + rid = random_uuid()

488 + server_label, tool_name = _parse_mcp_recipient(current_recipient)

489 + return [

490 + McpCall(

491 + arguments=parser.current_content,

492 + type="mcp_call",

493 + name=tool_name,

494 + server_label=server_label,

495 + id=f"mcp_{rid}",

496 + status="in_progress",

497 + )

498 + ]

499 +

500 + if parser.current_channel == "commentary":

501 + # Per Harmony format, preambles (commentary with no recipient) are

502 + # intended to be shown to end-users, unlike analysis channel content.

503 + output_text = ResponseOutputText(

504 + text=parser.current_content,

505 + annotations=[],

506 + type="output_text",

507 + logprobs=None,

508 + )

509 + return [

510 + ResponseOutputMessage(

511 + id=f"msg_{random_uuid()}",

512 + content=[output_text],

513 + role="assistant",

514 + status="incomplete",

515 + type="message",

516 + )

517 + ]

518 +

519 + if parser.current_channel == "analysis":

520 + return [

521 + ResponseReasoningItem(

522 + id=f"rs_{random_uuid()}",

523 + summary=[],

524 + type="reasoning",

525 + content=[

526 + ResponseReasoningTextContent(

527 + text=parser.current_content, type="reasoning_text"

528 + )

529 + ],

530 + status=None,

531 + )

532 + ]

533 +

534 + if parser.current_channel == "final":

535 + output_text = ResponseOutputText(

536 + text=parser.current_content,

537 + annotations=[], # TODO

538 + type="output_text",

539 + logprobs=None, # TODO

540 + )

541 + text_item = ResponseOutputMessage(

542 + id=f"msg_{random_uuid()}",

543 + content=[output_text],

544 + role="assistant",

545 + # if the parser still has messages (ie if the generator got cut

546 + # abruptly), this should be incomplete

547 + status="incomplete",

548 + type="message",

549 + )

550 + return [text_item]

551 +

552 + return []

vllm/entrypoints/openai/responses/serving.py

+11 -9

@@ -58,15 +58,11 @@ from vllm.entrypoints.openai.engine.serving import (

58 )

59 from vllm.entrypoints.openai.models.serving import OpenAIServingModels

60 from vllm.entrypoints.openai.parser.harmony_utils import (

61 - construct_harmony_previous_input_messages,

61 get_developer_message,

62 get_stop_tokens_for_assistant_actions,

63 get_system_message,

64 get_user_message,

65 has_custom_tools,

67 - parse_output_message,

68 - parse_remaining_state,

69 - parse_response_input,

66 render_for_completion,

67 )

68 from vllm.entrypoints.openai.responses.context import (

@@ -76,6 +72,12 @@ from vllm.entrypoints.openai.responses.context import (

72 SimpleContext,

73 StreamingHarmonyContext,

74 )

75 +from vllm.entrypoints.openai.responses.harmony import (

76 + construct_harmony_previous_input_messages,

77 + harmony_to_response_output,

78 + parser_state_to_response_output,

79 + response_input_to_harmony,

80 +)

81 from vllm.entrypoints.openai.responses.protocol import (

82 InputTokensDetails,

83 OutputTokensDetails,

@@ -954,9 +956,9 @@ class OpenAIServingResponses(OpenAIServing):

956 output_items: list[ResponseOutputItem] = []

957 num_init_messages = context.num_init_messages

958 for msg in context.messages[num_init_messages:]:

957 - output_items.extend(parse_output_message(msg))

959 + output_items.extend(harmony_to_response_output(msg))

960 # Handle the generation stopped in the middle (if any).

959 - last_items = parse_remaining_state(context.parser)

961 + last_items = parser_state_to_response_output(context.parser)

962 if last_items:

963 output_items.extend(last_items)

964 return output_items

@@ -1103,13 +1105,13 @@ class OpenAIServingResponses(OpenAIServing):

1105 else:

1106 prev_outputs = []

1107 for response_msg in request.input:

1106 - new_msg = parse_response_input(response_msg, prev_outputs)

1108 + new_msg = response_input_to_harmony(response_msg, prev_outputs)

1109 if new_msg.author.role != "system":

1110 messages.append(new_msg)

1111

1112 # User passes in a tool call request and its output. We need

1111 - # to add the tool call request to prev_outputs so that the

1112 - # parse_response_input can find the tool call request when

1113 + # to add the tool call request to prev_outputs so that

1114 + # response_input_to_harmony can find the tool call request when

1115 # parsing the tool call output.

1116 if isinstance(response_msg, ResponseFunctionToolCall):

1117 prev_outputs.append(response_msg)