| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Unit tests for vllm.entrypoints.openai.responses.harmony.""" |
| 4 | |
| 5 | import pytest |
| 6 | from openai.types.responses import ( |
| 7 | ResponseFunctionToolCall, |
| 8 | ResponseFunctionWebSearch, |
| 9 | ResponseOutputMessage, |
| 10 | ResponseReasoningItem, |
| 11 | ) |
| 12 | from openai.types.responses.response_output_item import McpCall |
| 13 | from openai_harmony import Author, Message, Role, TextContent |
| 14 | |
| 15 | from vllm.entrypoints.openai.responses.harmony import ( |
| 16 | harmony_to_response_output, |
| 17 | response_previous_input_to_harmony, |
| 18 | ) |
| 19 | from vllm.exceptions import VLLMValidationError |
| 20 | |
| 21 | |
| 22 | class TestResponsePreviousInputToHarmony: |
| 23 | """Tests for scenarios that are specific to the Responses API |
| 24 | response_previous_input_to_harmony function. |
| 25 | """ |
| 26 | |
| 27 | def test_message_with_empty_content(self): |
| 28 | """Test parsing message with empty string content.""" |
| 29 | chat_msg = { |
| 30 | "role": "user", |
| 31 | "content": "", |
| 32 | } |
| 33 | |
| 34 | messages = response_previous_input_to_harmony(chat_msg) |
| 35 | |
| 36 | assert len(messages) == 1 |
| 37 | assert messages[0].content[0].text == "" |
| 38 | |
| 39 | def test_tool_message_with_string_content(self): |
| 40 | """Test parsing tool message with string content.""" |
| 41 | chat_msg = { |
| 42 | "role": "tool", |
| 43 | "name": "get_weather", |
| 44 | "content": "The weather in San Francisco is sunny, 72°F", |
| 45 | } |
| 46 | |
| 47 | messages = response_previous_input_to_harmony(chat_msg) |
| 48 | |
| 49 | assert len(messages) == 1 |
| 50 | assert messages[0].author.role == Role.TOOL |
| 51 | assert messages[0].author.name == "functions.get_weather" |
| 52 | assert ( |
| 53 | messages[0].content[0].text == "The weather in San Francisco is sunny, 72°F" |
| 54 | ) |
| 55 | assert messages[0].channel == "commentary" |
| 56 | |
| 57 | def test_tool_message_with_array_content(self): |
| 58 | """Test parsing tool message with array content.""" |
| 59 | chat_msg = { |
| 60 | "role": "tool", |
| 61 | "name": "search_results", |
| 62 | "content": [ |
| 63 | {"type": "text", "text": "Result 1: "}, |
| 64 | {"type": "text", "text": "Result 2: "}, |
| 65 | { |
| 66 | "type": "image", |
| 67 | "url": "http://example.com/img.png", |
| 68 | }, # Should be ignored |
| 69 | {"type": "text", "text": "Result 3"}, |
| 70 | ], |
| 71 | } |
| 72 | |
| 73 | messages = response_previous_input_to_harmony(chat_msg) |
| 74 | |
| 75 | assert len(messages) == 1 |
| 76 | assert messages[0].author.role == Role.TOOL |
| 77 | assert messages[0].author.name == "functions.search_results" |
| 78 | assert messages[0].content[0].text == "Result 1: Result 2: Result 3" |
| 79 | |
| 80 | def test_tool_message_with_empty_content(self): |
| 81 | """Test parsing tool message with None content.""" |
| 82 | chat_msg = { |
| 83 | "role": "tool", |
| 84 | "name": "empty_tool", |
| 85 | "content": None, |
| 86 | } |
| 87 | |
| 88 | messages = response_previous_input_to_harmony(chat_msg) |
| 89 | |
| 90 | assert len(messages) == 1 |
| 91 | assert messages[0].author.role == Role.TOOL |
| 92 | assert messages[0].author.name == "functions.empty_tool" |
| 93 | assert messages[0].content[0].text == "" |
| 94 | |
| 95 | def test_chat_message_without_role_raises_validation_error(self): |
| 96 | """A chat-format message missing the 'role' key is invalid input.""" |
| 97 | chat_msg = {"content": "hello, no role here"} |
| 98 | |
| 99 | with pytest.raises( |
| 100 | VLLMValidationError, match="Message has no 'role' key" |
| 101 | ) as exc_info: |
| 102 | response_previous_input_to_harmony(chat_msg) |
| 103 | assert exc_info.value.parameter == "input" |
| 104 | |
| 105 | |
| 106 | class TestHarmonyToResponseOutput: |
| 107 | """Tests for harmony_to_response_output function.""" |
| 108 | |
| 109 | @pytest.mark.parametrize("incomplete", [False, True]) |
| 110 | def test_commentary_with_no_recipient_creates_message(self, incomplete): |
| 111 | """Test that commentary with recipient=None (preambles) creates message items. |
| 112 | |
| 113 | Per Harmony format, preambles are intended to be shown to end-users, |
| 114 | unlike analysis channel content which is hidden reasoning. |
| 115 | See: https://cookbook.openai.com/articles/openai-harmony |
| 116 | """ |
| 117 | message = Message.from_role_and_content( |
| 118 | Role.ASSISTANT, "I will now search for the weather information." |
| 119 | ) |
| 120 | message = message.with_channel("commentary") |
| 121 | # recipient is None by default, representing a preamble |
| 122 | |
| 123 | output_items = harmony_to_response_output( |
| 124 | message, frozenset(), incomplete=incomplete |
| 125 | ) |
| 126 | |
| 127 | assert len(output_items) == 1 |
| 128 | assert isinstance(output_items[0], ResponseOutputMessage) |
| 129 | assert output_items[0].type == "message" |
| 130 | assert output_items[0].role == "assistant" |
| 131 | assert output_items[0].status == ("incomplete" if incomplete else "completed") |
| 132 | assert len(output_items[0].content) == 1 |
| 133 | assert output_items[0].content[0].type == "output_text" |
| 134 | assert ( |
| 135 | output_items[0].content[0].text |
| 136 | == "I will now search for the weather information." |
| 137 | ) |
| 138 | |
| 139 | @pytest.mark.parametrize("channel", ["commentary", "comment", "analysis", "final"]) |
| 140 | @pytest.mark.parametrize( |
| 141 | ("recipient", "fn_names", "expected_name"), |
| 142 | [ |
| 143 | ("functions.get_weather", frozenset(), "get_weather"), |
| 144 | ("get_weather", frozenset({"get_weather"}), "get_weather"), |
| 145 | ("math.sum", frozenset({"math.sum"}), "math.sum"), |
| 146 | ], |
| 147 | ) |
| 148 | @pytest.mark.parametrize("incomplete", [False, True]) |
| 149 | def test_function_recipient_creates_function_call( |
| 150 | self, channel, recipient, fn_names, expected_name, incomplete |
| 151 | ): |
| 152 | """Function recipients create function calls across channels.""" |
| 153 | content = '{"location": "San Francisco"}' |
| 154 | if recipient == "math.sum": |
| 155 | content = '{"a": 1, "b": 2}' |
| 156 | |
| 157 | message = Message.from_role_and_content(Role.ASSISTANT, content) |
| 158 | message = message.with_channel(channel) |
| 159 | message = message.with_recipient(recipient) |
| 160 | |
| 161 | output_items = harmony_to_response_output( |
| 162 | message, fn_names, incomplete=incomplete |
| 163 | ) |
| 164 | |
| 165 | assert len(output_items) == 1 |
| 166 | assert isinstance(output_items[0], ResponseFunctionToolCall) |
| 167 | assert output_items[0].type == "function_call" |
| 168 | assert output_items[0].name == expected_name |
| 169 | assert output_items[0].arguments == content |
| 170 | assert output_items[0].call_id.startswith("call_") |
| 171 | assert output_items[0].id.startswith("fc_") |
| 172 | assert output_items[0].status == ("incomplete" if incomplete else "completed") |
| 173 | |
| 174 | @pytest.mark.parametrize("channel", ["commentary", "comment", "analysis", "final"]) |
| 175 | @pytest.mark.parametrize( |
| 176 | ("recipient", "content"), |
| 177 | [ |
| 178 | ("python", "import numpy as np\nprint(np.array([1, 2, 3]))"), |
| 179 | ("browser", "Navigating to the specified URL"), |
| 180 | ("container", "Running command in container"), |
| 181 | ], |
| 182 | ) |
| 183 | @pytest.mark.parametrize("incomplete", [False, True]) |
| 184 | def test_builtin_recipient_creates_reasoning( |
| 185 | self, channel, recipient, content, incomplete |
| 186 | ): |
| 187 | """Built-in recipients create reasoning items.""" |
| 188 | message = Message.from_role_and_content(Role.ASSISTANT, content) |
| 189 | message = message.with_channel(channel) |
| 190 | message = message.with_recipient(recipient) |
| 191 | |
| 192 | output_items = harmony_to_response_output( |
| 193 | message, frozenset(), incomplete=incomplete |
| 194 | ) |
| 195 | |
| 196 | assert len(output_items) == 1 |
| 197 | assert isinstance(output_items[0], ResponseReasoningItem) |
| 198 | assert output_items[0].type == "reasoning" |
| 199 | assert output_items[0].content[0].text == content |
| 200 | assert output_items[0].status is None |
| 201 | |
| 202 | @pytest.mark.parametrize("channel", ["commentary", "comment", "analysis", "final"]) |
| 203 | @pytest.mark.parametrize( |
| 204 | ("recipient", "fn_names", "content", "expected_name", "expected_server_label"), |
| 205 | [ |
| 206 | ( |
| 207 | "get_weather", |
| 208 | frozenset(), |
| 209 | '{"arg": "value"}', |
| 210 | "get_weather", |
| 211 | "get_weather", |
| 212 | ), |
| 213 | ( |
| 214 | "not_get_weather", |
| 215 | frozenset({"get_weather"}), |
| 216 | '{"arg": "value"}', |
| 217 | "not_get_weather", |
| 218 | "not_get_weather", |
| 219 | ), |
| 220 | ("repo_browser.list", frozenset(), '{"cmd": "ls"}', "list", "repo_browser"), |
| 221 | ], |
| 222 | ) |
| 223 | @pytest.mark.parametrize("incomplete", [False, True]) |
| 224 | def test_non_function_non_builtin_recipient_creates_mcp_call( |
| 225 | self, |
| 226 | channel, |
| 227 | recipient, |
| 228 | fn_names, |
| 229 | content, |
| 230 | expected_name, |
| 231 | expected_server_label, |
| 232 | incomplete, |
| 233 | ): |
| 234 | """Non-function, non-built-in recipients create MCP calls.""" |
| 235 | message = Message.from_role_and_content(Role.ASSISTANT, content) |
| 236 | message = message.with_channel(channel) |
| 237 | message = message.with_recipient(recipient) |
| 238 | |
| 239 | output_items = harmony_to_response_output( |
| 240 | message, fn_names, incomplete=incomplete |
| 241 | ) |
| 242 | |
| 243 | assert len(output_items) == 1 |
| 244 | assert isinstance(output_items[0], McpCall) |
| 245 | assert output_items[0].type == "mcp_call" |
| 246 | assert output_items[0].name == expected_name |
| 247 | assert output_items[0].server_label == expected_server_label |
| 248 | assert output_items[0].arguments == content |
| 249 | assert output_items[0].status == ("incomplete" if incomplete else "completed") |
| 250 | |
| 251 | @pytest.mark.parametrize("incomplete", [False, True]) |
| 252 | def test_browser_search_recipient_respects_incomplete(self, incomplete): |
| 253 | """browser.search emits a web search call unless the item is incomplete.""" |
| 254 | message = Message.from_role_and_content( |
| 255 | Role.ASSISTANT, '{"query": "weather in San Francisco"}' |
| 256 | ) |
| 257 | message = message.with_channel("commentary") |
| 258 | message = message.with_recipient("browser.search") |
| 259 | |
| 260 | output_items = harmony_to_response_output( |
| 261 | message, frozenset(), incomplete=incomplete |
| 262 | ) |
| 263 | |
| 264 | if incomplete: |
| 265 | assert output_items == [] |
| 266 | return |
| 267 | |
| 268 | assert len(output_items) == 1 |
| 269 | assert isinstance(output_items[0], ResponseFunctionWebSearch) |
| 270 | assert output_items[0].type == "web_search_call" |
| 271 | assert output_items[0].status == "completed" |
| 272 | assert output_items[0].action.type == "search" |
| 273 | assert output_items[0].action.query == "cursor:weather in San Francisco" |
| 274 | |
| 275 | def test_browser_find_recipient_uses_responses_action_type(self): |
| 276 | """browser.find must use the Responses API find_in_page action type.""" |
| 277 | message = ( |
| 278 | Message.from_role_and_content( |
| 279 | Role.ASSISTANT, '{"pattern": "vLLM", "cursor": 42}' |
| 280 | ) |
| 281 | .with_channel("analysis") |
| 282 | .with_recipient("browser.find") |
| 283 | ) |
| 284 | |
| 285 | output_items = harmony_to_response_output(message, frozenset()) |
| 286 | |
| 287 | assert len(output_items) == 1 |
| 288 | assert isinstance(output_items[0], ResponseFunctionWebSearch) |
| 289 | assert output_items[0].action.type == "find_in_page" |
| 290 | assert output_items[0].action.pattern == "vLLM" |
| 291 | |
| 292 | def test_commentary_with_empty_content_and_no_recipient(self): |
| 293 | """Test edge case: empty commentary with recipient=None.""" |
| 294 | message = Message.from_role_and_content(Role.ASSISTANT, "") |
| 295 | message = message.with_channel("commentary") |
| 296 | |
| 297 | output_items = harmony_to_response_output(message, frozenset()) |
| 298 | |
| 299 | assert len(output_items) == 1 |
| 300 | assert isinstance(output_items[0], ResponseOutputMessage) |
| 301 | assert output_items[0].content[0].text == "" |
| 302 | |
| 303 | def test_commentary_with_multiple_contents_and_no_recipient(self): |
| 304 | """Test multiple content items in commentary with no recipient.""" |
| 305 | contents = [ |
| 306 | TextContent(text="Step 1: Analyze the request"), |
| 307 | TextContent(text="Step 2: Prepare to call functions"), |
| 308 | ] |
| 309 | message = Message.from_role_and_contents(Role.ASSISTANT, contents) |
| 310 | message = message.with_channel("commentary") |
| 311 | |
| 312 | output_items = harmony_to_response_output(message, frozenset()) |
| 313 | |
| 314 | # _parse_final_message returns single ResponseOutputMessage with |
| 315 | # multiple contents |
| 316 | assert len(output_items) == 1 |
| 317 | assert isinstance(output_items[0], ResponseOutputMessage) |
| 318 | assert len(output_items[0].content) == 2 |
| 319 | assert output_items[0].content[0].text == "Step 1: Analyze the request" |
| 320 | assert output_items[0].content[1].text == "Step 2: Prepare to call functions" |
| 321 | |
| 322 | def test_commentary_with_multiple_function_calls(self): |
| 323 | """Test multiple function calls in commentary channel.""" |
| 324 | contents = [ |
| 325 | TextContent(text='{"location": "San Francisco"}'), |
| 326 | TextContent(text='{"location": "New York"}'), |
| 327 | ] |
| 328 | message = Message.from_role_and_contents(Role.ASSISTANT, contents) |
| 329 | message = message.with_channel("commentary") |
| 330 | message = message.with_recipient("functions.get_weather") |
| 331 | |
| 332 | output_items = harmony_to_response_output(message, frozenset()) |
| 333 | |
| 334 | assert len(output_items) == 2 |
| 335 | assert all(isinstance(item, ResponseFunctionToolCall) for item in output_items) |
| 336 | assert output_items[0].name == "get_weather" |
| 337 | assert output_items[1].name == "get_weather" |
| 338 | assert output_items[0].arguments == '{"location": "San Francisco"}' |
| 339 | assert output_items[1].arguments == '{"location": "New York"}' |
| 340 | |
| 341 | def test_analysis_channel_creates_reasoning(self): |
| 342 | """Test that analysis channel creates reasoning items.""" |
| 343 | message = Message.from_role_and_content( |
| 344 | Role.ASSISTANT, "Analyzing the problem step by step..." |
| 345 | ) |
| 346 | message = message.with_channel("analysis") |
| 347 | |
| 348 | output_items = harmony_to_response_output(message, frozenset()) |
| 349 | |
| 350 | assert len(output_items) == 1 |
| 351 | assert isinstance(output_items[0], ResponseReasoningItem) |
| 352 | assert output_items[0].type == "reasoning" |
| 353 | assert ( |
| 354 | output_items[0].content[0].text == "Analyzing the problem step by step..." |
| 355 | ) |
| 356 | |
| 357 | def test_non_assistant_message_returns_empty(self): |
| 358 | """Test that non-assistant messages return empty list. |
| 359 | |
| 360 | Per the implementation, tool messages to assistant (e.g., search results) |
| 361 | are not included in final output to align with OpenAI behavior. |
| 362 | """ |
| 363 | message = Message.from_author_and_content( |
| 364 | Author.new(Role.TOOL, "functions.get_weather"), |
| 365 | "The weather is sunny, 72°F", |
| 366 | ) |
| 367 | |
| 368 | output_items = harmony_to_response_output(message, frozenset()) |
| 369 | |
| 370 | assert len(output_items) == 0 |