@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8f8fda261a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 11, 15:00 UTC

[Bugfix] Multiple fixes for gpt-oss Chat Completion prompting (#28729)

Signed-off-by: Ben Browning <bbrownin@redhat.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>

Ben Browning committed Dec 11, 2025 at 23:59 UTC 8f8fda261a620234fdeea338f44093d5d8072879
6 files changed +1721 -111
tests/entrypoints/openai/parser/test_harmony_utils.py
+666 -85
@@ -1,21 +1,37 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +import pytest
5 from openai.types.responses import ResponseFunctionToolCall, ResponseReasoningItem
6 from openai.types.responses.response_output_item import McpCall
7 from openai_harmony import Author, Message, Role, TextContent
8
9 +from tests.entrypoints.openai.utils import verify_harmony_messages
10 from vllm.entrypoints.openai.parser.harmony_utils import (
11 + auto_drop_analysis_messages,
12 + get_encoding,
13 has_custom_tools,
14 + parse_chat_input_to_harmony_message,
15 + parse_chat_output,
16 parse_input_to_harmony_message,
17 parse_output_message,
18 )
19
20
15 -class TestParseInputToHarmonyMessage:
16 - """Tests for parse_input_to_harmony_message function."""
21 +class TestCommonParseInputToHarmonyMessage:
22 + """
23 + Tests for scenarios that are common to both Chat Completion
24 + parse_chat_input_to_harmony_message and Responsees API
25 + parse_input_to_harmony_message functions.
26 + """
27 +
28 + @pytest.fixture(
29 + params=[parse_chat_input_to_harmony_message, parse_input_to_harmony_message]
30 + )
31 + def parse_function(self, request):
32 + return request.param
33
18 - def test_assistant_message_with_tool_calls(self):
34 + def test_assistant_message_with_tool_calls(self, parse_function):
35 """Test parsing assistant message with tool calls."""
36 chat_msg = {
37 "role": "assistant",
@@ -35,7 +51,7 @@ class TestParseInputToHarmonyMessage:
51 ],
52 }
53
38 - messages = parse_input_to_harmony_message(chat_msg)
54 + messages = parse_function(chat_msg)
55
56 assert len(messages) == 2
57
@@ -53,7 +69,7 @@ class TestParseInputToHarmonyMessage:
69 assert messages[1].recipient == "functions.search_web"
70 assert messages[1].content_type == "json"
71
56 - def test_assistant_message_with_empty_tool_call_arguments(self):
72 + def test_assistant_message_with_empty_tool_call_arguments(self, parse_function):
73 """Test parsing assistant message with tool call having None arguments."""
74 chat_msg = {
75 "role": "assistant",
@@ -67,12 +83,152 @@ class TestParseInputToHarmonyMessage:
83 ],
84 }
85
70 - messages = parse_input_to_harmony_message(chat_msg)
86 + messages = parse_function(chat_msg)
87
88 assert len(messages) == 1
89 assert messages[0].content[0].text == ""
90 assert messages[0].recipient == "functions.get_current_time"
91
92 + def test_system_message(self, parse_function):
93 + """Test parsing system message."""
94 + chat_msg = {
95 + "role": "system",
96 + "content": "You are a helpful assistant",
97 + }
98 +
99 + messages = parse_function(chat_msg)
100 +
101 + assert len(messages) == 1
102 + # System messages are converted using Message.from_dict
103 + # which should preserve the role
104 + assert messages[0].author.role == Role.SYSTEM
105 +
106 + def test_developer_message(self, parse_function):
107 + """Test parsing developer message."""
108 + chat_msg = {
109 + "role": "developer",
110 + "content": "Use concise language",
111 + }
112 +
113 + messages = parse_function(chat_msg)
114 +
115 + assert len(messages) == 1
116 + assert messages[0].author.role == Role.DEVELOPER
117 +
118 + def test_user_message_with_string_content(self, parse_function):
119 + """Test parsing user message with string content."""
120 + chat_msg = {
121 + "role": "user",
122 + "content": "What's the weather in San Francisco?",
123 + }
124 +
125 + messages = parse_function(chat_msg)
126 +
127 + assert len(messages) == 1
128 + assert messages[0].author.role == Role.USER
129 + assert messages[0].content[0].text == "What's the weather in San Francisco?"
130 +
131 + def test_user_message_with_array_content(self, parse_function):
132 + """Test parsing user message with array content."""
133 + chat_msg = {
134 + "role": "user",
135 + "content": [
136 + {"text": "What's in this image? "},
137 + {"text": "Please describe it."},
138 + ],
139 + }
140 +
141 + messages = parse_function(chat_msg)
142 +
143 + assert len(messages) == 1
144 + assert messages[0].author.role == Role.USER
145 + assert len(messages[0].content) == 2
146 + assert messages[0].content[0].text == "What's in this image? "
147 + assert messages[0].content[1].text == "Please describe it."
148 +
149 + def test_assistant_message_with_string_content(self, parse_function):
150 + """Test parsing assistant message with string content (no tool calls)."""
151 + chat_msg = {
152 + "role": "assistant",
153 + "content": "Hello! How can I help you today?",
154 + }
155 +
156 + messages = parse_function(chat_msg)
157 +
158 + assert len(messages) == 1
159 + assert messages[0].author.role == Role.ASSISTANT
160 + assert messages[0].content[0].text == "Hello! How can I help you today?"
161 +
162 + def test_pydantic_model_input(self, parse_function):
163 + """Test parsing Pydantic model input (has model_dump method)."""
164 +
165 + class MockPydanticModel:
166 + def model_dump(self, exclude_none=True):
167 + return {
168 + "role": "user",
169 + "content": "Test message",
170 + }
171 +
172 + chat_msg = MockPydanticModel()
173 + messages = parse_function(chat_msg)
174 +
175 + assert len(messages) == 1
176 + assert messages[0].author.role == Role.USER
177 + assert messages[0].content[0].text == "Test message"
178 +
179 + def test_tool_call_with_missing_function_fields(self, parse_function):
180 + """Test parsing tool call with missing name or arguments."""
181 + chat_msg = {
182 + "role": "assistant",
183 + "tool_calls": [
184 + {
185 + "function": {} # Missing both name and arguments
186 + }
187 + ],
188 + }
189 +
190 + messages = parse_function(chat_msg)
191 +
192 + assert len(messages) == 1
193 + assert messages[0].recipient == "functions."
194 + assert messages[0].content[0].text == ""
195 +
196 + def test_array_content_with_missing_text(self, parse_function):
197 + """Test parsing array content where text field is missing."""
198 + chat_msg = {
199 + "role": "user",
200 + "content": [
201 + {}, # Missing text field
202 + {"text": "actual text"},
203 + ],
204 + }
205 +
206 + messages = parse_function(chat_msg)
207 +
208 + assert len(messages) == 1
209 + assert len(messages[0].content) == 2
210 + assert messages[0].content[0].text == ""
211 + assert messages[0].content[1].text == "actual text"
212 +
213 +
214 +class TestParseInputToHarmonyMessage:
215 + """
216 + Tests for scenarios that are specific to the Responses API
217 + parse_input_to_harmony_message function.
218 + """
219 +
220 + def test_message_with_empty_content(self):
221 + """Test parsing message with empty string content."""
222 + chat_msg = {
223 + "role": "user",
224 + "content": "",
225 + }
226 +
227 + messages = parse_input_to_harmony_message(chat_msg)
228 +
229 + assert len(messages) == 1
230 + assert messages[0].content[0].text == ""
231 +
232 def test_tool_message_with_string_content(self):
233 """Test parsing tool message with string content."""
234 chat_msg = {
@@ -111,6 +267,7 @@ class TestParseInputToHarmonyMessage:
267
268 assert len(messages) == 1
269 assert messages[0].author.role == Role.TOOL
270 + assert messages[0].author.name == "functions.search_results"
271 assert messages[0].content[0].text == "Result 1: Result 2: Result 3"
272
273 def test_tool_message_with_empty_content(self):
@@ -124,140 +281,564 @@ class TestParseInputToHarmonyMessage:
281 messages = parse_input_to_harmony_message(chat_msg)
282
283 assert len(messages) == 1
284 + assert messages[0].author.role == Role.TOOL
285 + assert messages[0].author.name == "functions.empty_tool"
286 assert messages[0].content[0].text == ""
287
129 - def test_system_message(self):
130 - """Test parsing system message."""
288 +
289 +class TestParseChatInputToHarmonyMessage:
290 + """
291 + Tests for scenarios that are specific to the Chat Completion API
292 + parse_chat_input_to_harmony_message function.
293 + """
294 +
295 + def test_user_message_with_empty_content(self):
296 chat_msg = {
132 - "role": "system",
133 - "content": "You are a helpful assistant",
297 + "role": "user",
298 + "content": "",
299 }
300
136 - messages = parse_input_to_harmony_message(chat_msg)
301 + messages = parse_chat_input_to_harmony_message(chat_msg)
302
138 - assert len(messages) == 1
139 - # System messages are converted using Message.from_dict
140 - # which should preserve the role
141 - assert messages[0].author.role == Role.SYSTEM
303 + verify_harmony_messages(
304 + messages,
305 + [
306 + {
307 + "role": "user",
308 + "content": "",
309 + },
310 + ],
311 + )
312
143 - def test_developer_message(self):
144 - """Test parsing developer message."""
313 + def test_user_message_with_none_content(self):
314 chat_msg = {
146 - "role": "developer",
147 - "content": "Use concise language",
315 + "role": "user",
316 + "content": None,
317 }
318
150 - messages = parse_input_to_harmony_message(chat_msg)
319 + messages = parse_chat_input_to_harmony_message(chat_msg)
320
152 - assert len(messages) == 1
153 - assert messages[0].author.role == Role.DEVELOPER
321 + verify_harmony_messages(
322 + messages,
323 + [
324 + {
325 + "role": "user",
326 + "content": "",
327 + },
328 + ],
329 + )
330
155 - def test_user_message_with_string_content(self):
156 - """Test parsing user message with string content."""
331 + def test_assistant_message_with_empty_content(self):
332 chat_msg = {
158 - "role": "user",
159 - "content": "What's the weather in San Francisco?",
333 + "role": "assistant",
334 + "content": "",
335 }
336
162 - messages = parse_input_to_harmony_message(chat_msg)
337 + messages = parse_chat_input_to_harmony_message(chat_msg)
338
164 - assert len(messages) == 1
165 - assert messages[0].author.role == Role.USER
166 - assert messages[0].content[0].text == "What's the weather in San Francisco?"
339 + assert len(messages) == 0
340
168 - def test_user_message_with_array_content(self):
169 - """Test parsing user message with array content."""
341 + def test_assistant_message_with_none_content(self):
342 chat_msg = {
171 - "role": "user",
172 - "content": [
173 - {"text": "What's in this image? "},
174 - {"text": "Please describe it."},
343 + "role": "assistant",
344 + "content": None,
345 + }
346 +
347 + messages = parse_chat_input_to_harmony_message(chat_msg)
348 +
349 + assert len(messages) == 0
350 +
351 + def test_assistant_message_with_content_but_empty_reasoning(self):
352 + chat_msg = {
353 + "role": "assistant",
354 + "content": "The answer is 4.",
355 + "reasoning": "",
356 + }
357 +
358 + messages = parse_chat_input_to_harmony_message(chat_msg)
359 +
360 + verify_harmony_messages(
361 + messages,
362 + [
363 + {
364 + "role": "assistant",
365 + "channel": "final",
366 + "content": "The answer is 4.",
367 + },
368 ],
369 + )
370 +
371 + def test_assistant_message_with_reasoning_but_empty_content(self):
372 + chat_msg = {
373 + "role": "assistant",
374 + "reasoning": "I'm thinking about the user's question.",
375 + "content": "",
376 }
377
178 - messages = parse_input_to_harmony_message(chat_msg)
378 + messages = parse_chat_input_to_harmony_message(chat_msg)
379
180 - assert len(messages) == 1
181 - assert messages[0].author.role == Role.USER
182 - assert len(messages[0].content) == 2
183 - assert messages[0].content[0].text == "What's in this image? "
184 - assert messages[0].content[1].text == "Please describe it."
380 + verify_harmony_messages(
381 + messages,
382 + [
383 + {
384 + "role": "assistant",
385 + "channel": "analysis",
386 + "content": "I'm thinking about the user's question.",
387 + },
388 + ],
389 + )
390
186 - def test_assistant_message_with_string_content(self):
187 - """Test parsing assistant message with string content (no tool calls)."""
391 + def test_assistant_message_with_reasoning_but_none_content(self):
392 chat_msg = {
393 "role": "assistant",
190 - "content": "Hello! How can I help you today?",
394 + "reasoning": "I'm thinking about the user's question.",
395 + "content": None,
396 }
397
193 - messages = parse_input_to_harmony_message(chat_msg)
398 + messages = parse_chat_input_to_harmony_message(chat_msg)
399
195 - assert len(messages) == 1
196 - assert messages[0].author.role == Role.ASSISTANT
197 - assert messages[0].content[0].text == "Hello! How can I help you today?"
400 + verify_harmony_messages(
401 + messages,
402 + [
403 + {
404 + "role": "assistant",
405 + "channel": "analysis",
406 + "content": "I'm thinking about the user's question.",
407 + },
408 + ],
409 + )
410
199 - def test_pydantic_model_input(self):
200 - """Test parsing Pydantic model input (has model_dump method)."""
411 + def test_assistant_message_with_tool_calls_but_no_content(self):
412 + chat_msg = {
413 + "role": "assistant",
414 + "tool_calls": [
415 + {
416 + "function": {
417 + "name": "get_weather",
418 + "arguments": '{"location": "San Francisco"}',
419 + }
420 + }
421 + ],
422 + }
423
202 - class MockPydanticModel:
203 - def model_dump(self, exclude_none=True):
204 - return {
205 - "role": "user",
206 - "content": "Test message",
424 + messages = parse_chat_input_to_harmony_message(chat_msg)
425 +
426 + verify_harmony_messages(
427 + messages,
428 + [
429 + {
430 + "role": "assistant",
431 + "channel": "commentary",
432 + "recipient": "functions.get_weather",
433 + "content": '{"location": "San Francisco"}',
434 + "content_type": "json",
435 + },
436 + ],
437 + )
438 +
439 + def test_assistant_message_with_tool_calls_and_content(self):
440 + chat_msg = {
441 + "role": "assistant",
442 + "tool_calls": [
443 + {
444 + "function": {
445 + "name": "get_weather",
446 + "arguments": '{"location": "San Francisco"}',
447 + }
448 }
449 + ],
450 + "content": "I'll call the tool.",
451 + }
452
209 - chat_msg = MockPydanticModel()
210 - messages = parse_input_to_harmony_message(chat_msg)
453 + messages = parse_chat_input_to_harmony_message(chat_msg)
454
212 - assert len(messages) == 1
213 - assert messages[0].author.role == Role.USER
214 - assert messages[0].content[0].text == "Test message"
455 + verify_harmony_messages(
456 + messages,
457 + [
458 + {
459 + "role": "assistant",
460 + "channel": "commentary",
461 + "content": "I'll call the tool.",
462 + },
463 + {
464 + "role": "assistant",
465 + "channel": "commentary",
466 + "recipient": "functions.get_weather",
467 + "content": '{"location": "San Francisco"}',
468 + "content_type": "json",
469 + },
470 + ],
471 + )
472
216 - def test_message_with_empty_content(self):
217 - """Test parsing message with empty string content."""
473 + def test_assistant_message_with_tool_calls_and_reasoning(self):
474 chat_msg = {
219 - "role": "user",
220 - "content": "",
475 + "role": "assistant",
476 + "tool_calls": [
477 + {
478 + "function": {
479 + "name": "get_weather",
480 + "arguments": '{"location": "San Francisco"}',
481 + }
482 + }
483 + ],
484 + "reasoning": "I should use the get_weather tool.",
485 }
486
223 - messages = parse_input_to_harmony_message(chat_msg)
487 + messages = parse_chat_input_to_harmony_message(chat_msg)
488
225 - assert len(messages) == 1
226 - assert messages[0].content[0].text == ""
489 + verify_harmony_messages(
490 + messages,
491 + [
492 + {
493 + "role": "assistant",
494 + "channel": "analysis",
495 + "content": "I should use the get_weather tool.",
496 + },
497 + {
498 + "role": "assistant",
499 + "channel": "commentary",
500 + "recipient": "functions.get_weather",
501 + "content": '{"location": "San Francisco"}',
502 + "content_type": "json",
503 + },
504 + ],
505 + )
506
228 - def test_tool_call_with_missing_function_fields(self):
229 - """Test parsing tool call with missing name or arguments."""
507 + def test_assistant_message_with_tool_calls_and_reasoning_and_content(self):
508 chat_msg = {
509 "role": "assistant",
510 "tool_calls": [
511 {
234 - "function": {} # Missing both name and arguments
512 + "function": {
513 + "name": "get_weather",
514 + "arguments": '{"location": "San Francisco"}',
515 + }
516 }
517 ],
518 + "reasoning": "I should use the get_weather tool.",
519 + "content": "I'll call the tool.",
520 }
521
239 - messages = parse_input_to_harmony_message(chat_msg)
522 + messages = parse_chat_input_to_harmony_message(chat_msg)
523
241 - assert len(messages) == 1
242 - assert messages[0].recipient == "functions."
243 - assert messages[0].content[0].text == ""
524 + verify_harmony_messages(
525 + messages,
526 + [
527 + {
528 + "role": "assistant",
529 + "channel": "commentary",
530 + "content": "I'll call the tool.",
531 + },
532 + {
533 + "role": "assistant",
534 + "channel": "analysis",
535 + "content": "I should use the get_weather tool.",
536 + },
537 + {
538 + "role": "assistant",
539 + "channel": "commentary",
540 + "recipient": "functions.get_weather",
541 + "content": '{"location": "San Francisco"}',
542 + "content_type": "json",
543 + },
544 + ],
545 + )
546
245 - def test_array_content_with_missing_text(self):
246 - """Test parsing array content where text field is missing."""
547 + def test_tool_message_with_string_content(self):
548 + tool_id_names = {
549 + "call_123": "get_weather",
550 + }
551 chat_msg = {
248 - "role": "user",
552 + "role": "tool",
553 + "tool_call_id": "call_123",
554 + "content": "The weather in San Francisco is sunny, 72°F",
555 + }
556 +
557 + messages = parse_chat_input_to_harmony_message(
558 + chat_msg, tool_id_names=tool_id_names
559 + )
560 +
561 + verify_harmony_messages(
562 + messages,
563 + [
564 + {
565 + "role": "tool",
566 + "name": "functions.get_weather",
567 + "content": "The weather in San Francisco is sunny, 72°F",
568 + "channel": "commentary",
569 + },
570 + ],
571 + )
572 +
573 + def test_tool_message_with_array_content(self):
574 + tool_id_names = {
575 + "call_123": "search_results",
576 + }
577 + chat_msg = {
578 + "role": "tool",
579 + "tool_call_id": "call_123",
580 "content": [
250 - {}, # Missing text field
251 - {"text": "actual text"},
581 + {"type": "text", "text": "Result 1: "},
582 + {"type": "text", "text": "Result 2: "},
583 + {
584 + "type": "image",
585 + "url": "http://example.com/img.png",
586 + }, # Should be ignored
587 + {"type": "text", "text": "Result 3"},
588 ],
589 }
590
255 - messages = parse_input_to_harmony_message(chat_msg)
591 + messages = parse_chat_input_to_harmony_message(
592 + chat_msg, tool_id_names=tool_id_names
593 + )
594
257 - assert len(messages) == 1
258 - assert len(messages[0].content) == 2
259 - assert messages[0].content[0].text == ""
260 - assert messages[0].content[1].text == "actual text"
595 + verify_harmony_messages(
596 + messages,
597 + [
598 + {
599 + "role": "tool",
600 + "name": "functions.search_results",
601 + "content": "Result 1: Result 2: Result 3",
602 + "channel": "commentary",
603 + },
604 + ],
605 + )
606 +
607 + def test_tool_message_with_empty_content(self):
608 + tool_id_names = {
609 + "call_123": "empty_tool",
610 + }
611 + chat_msg = {
612 + "role": "tool",
613 + "tool_call_id": "call_123",
614 + "content": "",
615 + }
616 +
617 + messages = parse_chat_input_to_harmony_message(
618 + chat_msg, tool_id_names=tool_id_names
619 + )
620 +
621 + verify_harmony_messages(
622 + messages,
623 + [
624 + {
625 + "role": "tool",
626 + "name": "functions.empty_tool",
627 + "content": "",
628 + "channel": "commentary",
629 + },
630 + ],
631 + )
632 +
633 + def test_tool_message_with_none_content(self):
634 + tool_id_names = {
635 + "call_123": "empty_tool",
636 + }
637 + chat_msg = {
638 + "role": "tool",
639 + "tool_call_id": "call_123",
640 + "content": None,
641 + }
642 +
643 + messages = parse_chat_input_to_harmony_message(
644 + chat_msg, tool_id_names=tool_id_names
645 + )
646 +
647 + verify_harmony_messages(
648 + messages,
649 + [
650 + {
651 + "role": "tool",
652 + "name": "functions.empty_tool",
653 + "content": "",
654 + "channel": "commentary",
655 + },
656 + ],
657 + )
658 +
659 +
660 +class TestAutoDropAnalysisMessages:
661 + def test_no_analysis_messages(self) -> None:
662 + messages = [
663 + Message.from_role_and_content(
664 + Role.ASSISTANT, "The answer is 4."
665 + ).with_channel("final"),
666 + ]
667 + cleaned_messages = auto_drop_analysis_messages(messages)
668 + assert cleaned_messages == messages
669 +
670 + def test_only_analysis_message(self) -> None:
671 + messages = [
672 + Message.from_role_and_content(
673 + Role.ASSISTANT, "I'm thinking about the user's question."
674 + ).with_channel("analysis"),
675 + ]
676 + cleaned_messages = auto_drop_analysis_messages(messages)
677 + assert cleaned_messages == messages
678 +
679 + def test_multiple_analysis_messages_without_final_message(self) -> None:
680 + messages = [
681 + Message.from_role_and_content(
682 + Role.ASSISTANT, "I'm thinking about the user's question."
683 + ).with_channel("analysis"),
684 + Message.from_role_and_content(
685 + Role.ASSISTANT, "I'm thinking more."
686 + ).with_channel("analysis"),
687 + Message.from_role_and_content(
688 + Role.ASSISTANT, "I'm thinking even more."
689 + ).with_channel("analysis"),
690 + ]
691 + cleaned_messages = auto_drop_analysis_messages(messages)
692 + assert cleaned_messages == messages
693 +
694 + def test_only_final_message(self) -> None:
695 + messages = [
696 + Message.from_role_and_content(
697 + Role.ASSISTANT, "The answer is 4."
698 + ).with_channel("final"),
699 + ]
700 + cleaned_messages = auto_drop_analysis_messages(messages)
701 + assert cleaned_messages == messages
702 +
703 + def test_drops_one_analysis_messages_before_final_message(self) -> None:
704 + messages = [
705 + Message.from_role_and_content(
706 + Role.ASSISTANT, "I'm thinking about the user's question."
707 + ).with_channel("analysis"),
708 + Message.from_role_and_content(
709 + Role.ASSISTANT, "The answer is 4."
710 + ).with_channel("final"),
711 + Message.from_role_and_content(
712 + Role.ASSISTANT, "I should think harder."
713 + ).with_channel("analysis"),
714 + ]
715 + cleaned_messages = auto_drop_analysis_messages(messages)
716 + # Should have dropped the first analysis message
717 + assert cleaned_messages == messages[1:]
718 +
719 + def test_drops_all_analysis_messages_before_final_message(self) -> None:
720 + messages = [
721 + Message.from_role_and_content(
722 + Role.ASSISTANT, "I'm thinking about the user's question."
723 + ).with_channel("analysis"),
724 + Message.from_role_and_content(
725 + Role.ASSISTANT, "I'm thinking more."
726 + ).with_channel("analysis"),
727 + Message.from_role_and_content(
728 + Role.ASSISTANT, "I'm thinking even more."
729 + ).with_channel("analysis"),
730 + Message.from_role_and_content(
731 + Role.ASSISTANT, "The answer is 4."
732 + ).with_channel("final"),
733 + Message.from_role_and_content(
734 + Role.ASSISTANT, "I should think harder."
735 + ).with_channel("analysis"),
736 + ]
737 + cleaned_messages = auto_drop_analysis_messages(messages)
738 + # Should have dropped the first 3 analysis messages
739 + assert cleaned_messages == messages[3:]
740 +
741 + def test_multiple_analysis_messages_with_multiple_final_messages(self) -> None:
742 + messages = [
743 + Message.from_role_and_content(
744 + Role.ASSISTANT, "I'm thinking about the user's question."
745 + ).with_channel("analysis"),
746 + Message.from_role_and_content(
747 + Role.ASSISTANT, "I'm thinking more."
748 + ).with_channel("analysis"),
749 + Message.from_role_and_content(
750 + Role.ASSISTANT, "I'm thinking even more."
751 + ).with_channel("analysis"),
752 + Message.from_role_and_content(
753 + Role.ASSISTANT, "The answer is 4."
754 + ).with_channel("final"),
755 + Message.from_role_and_content(
756 + Role.ASSISTANT, "I should think harder."
757 + ).with_channel("analysis"),
758 + Message.from_role_and_content(
759 + Role.ASSISTANT, "The answer is 5."
760 + ).with_channel("final"),
761 + ]
762 + cleaned_messages = auto_drop_analysis_messages(messages)
763 + # Should have dropped all those analysis messages
764 + assert len(cleaned_messages) == 2
765 + assert cleaned_messages[0].content[0].text == "The answer is 4."
766 + assert cleaned_messages[1].content[0].text == "The answer is 5."
767 +
768 + def test_drops_non_assistant_analysis_messages(self) -> None:
769 + messages = [
770 + Message.from_role_and_content(
771 + Role.TOOL, "The tool thinks we should think harder."
772 + ).with_channel("analysis"),
773 + Message.from_role_and_content(
774 + Role.ASSISTANT, "The answer is 4."
775 + ).with_channel("final"),
776 + ]
777 + cleaned_messages = auto_drop_analysis_messages(messages)
778 + # Should have dropped the analysis message
779 + assert cleaned_messages == messages[1:]
780 +
781 +
782 +class TestParseChatOutput:
783 + def test_parse_chat_output_interrupted_first_message(self) -> None:
784 + harmony_str = "<|channel|>final<|message|>I'm in the middle of answering"
785 + token_ids = get_encoding().encode(harmony_str, allowed_special="all")
786 + reasoning, final_content, _ = parse_chat_output(token_ids)
787 + assert reasoning is None
788 + assert final_content == "I'm in the middle of answering"
789 +
790 + def test_parse_chat_output_interrupted_reasoning_first_message(self) -> None:
791 + harmony_str = "<|channel|>analysis<|message|>I'm in the middle of thinking"
792 + token_ids = get_encoding().encode(harmony_str, allowed_special="all")
793 + reasoning, final_content, _ = parse_chat_output(token_ids)
794 + assert reasoning == "I'm in the middle of thinking"
795 + assert final_content is None
796 +
797 + def test_parse_chat_output_complete_reasoning_interrupted_content(self) -> None:
798 + harmony_str = (
799 + "<|channel|>analysis<|message|>I'm thinking.<|end|>"
800 + "<|start|>assistant<|channel|>final"
801 + "<|message|>I'm in the middle of answering"
802 + )
803 + token_ids = get_encoding().encode(harmony_str, allowed_special="all")
804 + reasoning, final_content, _ = parse_chat_output(token_ids)
805 + assert reasoning == "I'm thinking."
806 + assert final_content == "I'm in the middle of answering"
807 +
808 + def test_parse_chat_output_complete_content(self) -> None:
809 + harmony_str = "<|channel|>final<|message|>The answer is 4.<|end|>"
810 + token_ids = get_encoding().encode(harmony_str, allowed_special="all")
811 + reasoning, final_content, _ = parse_chat_output(token_ids)
812 + assert reasoning is None
813 + assert final_content == "The answer is 4."
814 +
815 + def test_parse_chat_output_complete_commentary(self) -> None:
816 + harmony_str = (
817 + "<|channel|>commentary<|message|>I need to call some tools.<|end|>"
818 + )
819 + token_ids = get_encoding().encode(harmony_str, allowed_special="all")
820 + reasoning, final_content, _ = parse_chat_output(token_ids)
821 + assert reasoning is None
822 + assert final_content == "I need to call some tools."
823 +
824 + def test_parse_chat_output_complete_reasoning(self) -> None:
825 + harmony_str = (
826 + "<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
827 + )
828 + token_ids = get_encoding().encode(harmony_str, allowed_special="all")
829 + reasoning, final_content, _ = parse_chat_output(token_ids)
830 + assert reasoning == "I've thought hard about this."
831 + assert final_content is None
832 +
833 + def test_parse_chat_output_complete_reasoning_and_content(self) -> None:
834 + harmony_str = (
835 + "<|channel|>analysis<|message|>I've thought hard about this.<|end|>"
836 + "<|start|>assistant<|channel|>final<|message|>The answer is 4.<|end|>"
837 + )
838 + token_ids = get_encoding().encode(harmony_str, allowed_special="all")
839 + reasoning, final_content, _ = parse_chat_output(token_ids)
840 + assert reasoning == "I've thought hard about this."
841 + assert final_content == "The answer is 4."
842
843
844 class TestParseOutputMessage:
tests/entrypoints/openai/test_serving_chat.py
+645 -1
@@ -11,13 +11,25 @@ import pytest_asyncio
11 from openai import OpenAI
12
13 from vllm.config.multimodal import MultiModalConfig
14 -from vllm.entrypoints.openai.protocol import ChatCompletionRequest
14 +from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
15 +from vllm.entrypoints.openai.protocol import (
16 + ChatCompletionRequest,
17 + ChatCompletionResponse,
18 + RequestResponseMetadata,
19 +)
20 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
21 from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
22 +from vllm.entrypoints.openai.tool_parsers import ToolParserManager
23 +from vllm.outputs import CompletionOutput, RequestOutput
24 from vllm.tokenizers import get_tokenizer
25 from vllm.v1.engine.async_llm import AsyncLLM
26
27 from ...utils import RemoteOpenAIServer
28 +from .utils import (
29 + accumulate_streaming_response,
30 + verify_chat_response,
31 + verify_harmony_messages,
32 +)
33
34 GPT_OSS_MODEL_NAME = "openai/gpt-oss-20b"
35
@@ -728,3 +740,635 @@ async def test_serving_chat_data_parallel_rank_extraction():
740 # Verify that data_parallel_rank defaults to None
741 assert "data_parallel_rank" in mock_engine.generate.call_args.kwargs
742 assert mock_engine.generate.call_args.kwargs["data_parallel_rank"] is None
743 +
744 +
745 +class TestServingChatWithHarmony:
746 + """
747 + These tests ensure Chat Completion requests are being properly converted into
748 + Harmony messages and Harmony response messages back into Chat Completion responses.
749 + These tests are not exhaustive, but each one was created to cover a specific case
750 + that we got wrong but is now fixed.
751 +
752 + Any changes to the tests and their expectations may result in changes to the
753 + accuracy of model prompting and responses generated. It is suggested to run
754 + an evaluation or benchmarking suite (such as bfcl multi_turn) to understand
755 + any impact of changes in how we prompt Harmony models.
756 + """
757 +
758 + @pytest.fixture(params=[False, True], ids=["non_streaming", "streaming"])
759 + def stream(self, request) -> bool:
760 + """Parameterize tests to run in both non-streaming and streaming modes."""
761 + return request.param
762 +
763 + @pytest.fixture()
764 + def mock_engine(self) -> AsyncLLM:
765 + mock_engine = MagicMock(spec=AsyncLLM)
766 + mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME)
767 + mock_engine.errored = False
768 + mock_engine.model_config = MockModelConfig()
769 + mock_engine.input_processor = MagicMock()
770 + mock_engine.io_processor = MagicMock()
771 + return mock_engine
772 +
773 + @pytest.fixture()
774 + def serving_chat(self, mock_engine) -> OpenAIServingChat:
775 + chat = _build_serving_chat(mock_engine)
776 + chat.use_harmony = True
777 + chat.tool_parser = ToolParserManager.get_tool_parser("openai")
778 + return chat
779 +
780 + def mock_request_output_from_req_and_token_ids(
781 + self, req: ChatCompletionRequest, token_ids: list[int], finished: bool = False
782 + ) -> RequestOutput:
783 + # Our tests don't use most fields, so just get the token ids correct
784 + completion_output = CompletionOutput(
785 + index=0,
786 + text="",
787 + token_ids=token_ids,
788 + cumulative_logprob=0.0,
789 + logprobs=None,
790 + )
791 + return RequestOutput(
792 + request_id=req.request_id,
793 + prompt=[],
794 + prompt_token_ids=[],
795 + prompt_logprobs=None,
796 + outputs=[completion_output],
797 + finished=finished,
798 + )
799 +
800 + @pytest.fixture
801 + def weather_tools(self) -> list[dict[str, Any]]:
802 + return [
803 + {
804 + "type": "function",
805 + "function": {
806 + "name": "get_weather",
807 + "description": "Get the weather in a given location",
808 + "parameters": {
809 + "type": "object",
810 + "properties": {
811 + "location": {"type": "string"},
812 + },
813 + "required": ["location"],
814 + },
815 + },
816 + },
817 + ]
818 +
819 + @pytest.fixture
820 + def weather_messages_start(self) -> list[dict[str, Any]]:
821 + return [
822 + {
823 + "role": "user",
824 + "content": "What's the weather like in Paris today?",
825 + },
826 + ]
827 +
828 + async def generate_response_from_harmony_str(
829 + self,
830 + serving_chat: OpenAIServingChat,
831 + req: ChatCompletionRequest,
832 + harmony_str: str,
833 + stream: bool = False,
834 + ) -> ChatCompletionResponse:
835 + harmony_token_ids = get_encoding().encode(harmony_str, allowed_special="all")
836 +
837 + async def result_generator():
838 + if stream:
839 + for token_id in harmony_token_ids:
840 + yield self.mock_request_output_from_req_and_token_ids(
841 + req, [token_id]
842 + )
843 + yield self.mock_request_output_from_req_and_token_ids(
844 + req, [], finished=True
845 + )
846 + else:
847 + yield self.mock_request_output_from_req_and_token_ids(
848 + req, harmony_token_ids, finished=True
849 + )
850 +
851 + generator_func = (
852 + serving_chat.chat_completion_stream_generator
853 + if stream
854 + else serving_chat.chat_completion_full_generator
855 + )
856 +
857 + result = generator_func(
858 + request=req,
859 + result_generator=result_generator(),
860 + request_id=req.request_id,
861 + model_name=req.model,
862 + conversation=[],
863 + tokenizer=get_tokenizer(req.model),
864 + request_metadata=RequestResponseMetadata(
865 + request_id=req.request_id,
866 + model_name=req.model,
867 + ),
868 + )
869 +
870 + if stream:
871 + return await accumulate_streaming_response(result)
872 + return await result
873 +
874 + @pytest.mark.asyncio
875 + async def test_simple_chat(self, serving_chat, stream):
876 + messages = [{"role": "user", "content": "what is 1+1?"}]
877 +
878 + # Test the Harmony messages for the first turn's input
879 + req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
880 + input_messages, _, _ = serving_chat._make_request_with_harmony(req)
881 + verify_harmony_messages(
882 + input_messages,
883 + [
884 + {"role": "system"},
885 + {"role": "developer"},
886 + {"role": "user", "content": messages[0]["content"]},
887 + ],
888 + )
889 +
890 + # Test the Chat Completion response for the first turn's output
891 + reasoning_str = "We need to think really hard about this."
892 + final_str = "The answer is 2."
893 + response_str = (
894 + f"<|channel|>analysis<|message|>{reasoning_str}<|end|>"
895 + f"<|start|>assistant<|channel|>final<|message|>{final_str}<|end|>"
896 + )
897 + response = await self.generate_response_from_harmony_str(
898 + serving_chat, req, response_str, stream=stream
899 + )
900 + verify_chat_response(response, content=final_str, reasoning=reasoning_str)
901 +
902 + # Add the output messages from the first turn as input to the second turn
903 + for choice in response.choices:
904 + messages.append(choice.message.model_dump(exclude_none=True))
905 +
906 + # Test the Harmony messages for the second turn's input
907 + req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
908 + input_messages_2, _, _ = serving_chat._make_request_with_harmony(req_2)
909 + verify_harmony_messages(
910 + input_messages_2,
911 + [
912 + {"role": "system"},
913 + {"role": "developer"},
914 + {"role": "user"},
915 + # The analysis message should be dropped on subsequent inputs because
916 + # of the subsequent assistant message to the final channel.
917 + {"role": "assistant", "channel": "final", "content": final_str},
918 + ],
919 + )
920 +
921 + @pytest.mark.asyncio
922 + async def test_tool_call_response_with_content(
923 + self, serving_chat, stream, weather_tools, weather_messages_start
924 + ):
925 + tools = weather_tools
926 + messages = list(weather_messages_start)
927 +
928 + # Test the Harmony messages for the first turn's input
929 + req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
930 + input_messages, _, _ = serving_chat._make_request_with_harmony(req)
931 + verify_harmony_messages(
932 + input_messages,
933 + [
934 + {"role": "system"},
935 + {"role": "developer", "tool_definitions": ["get_weather"]},
936 + {"role": "user", "content": messages[0]["content"]},
937 + ],
938 + )
939 +
940 + # Test the Chat Completion response for the first turn's output
941 + commentary_str = "We'll call get_weather."
942 + tool_args_str = '{"location": "Paris"}'
943 + response_str = (
944 + f"<|channel|>commentary<|message|>{commentary_str}<|end|>"
945 + "<|start|>assistant to=functions.get_weather<|channel|>commentary"
946 + f"<|constrain|>json<|message|>{tool_args_str}<|call|>"
947 + )
948 + response = await self.generate_response_from_harmony_str(
949 + serving_chat, req, response_str, stream=stream
950 + )
951 + verify_chat_response(
952 + response,
953 + content=commentary_str,
954 + tool_calls=[("get_weather", tool_args_str)],
955 + )
956 +
957 + tool_call = response.choices[0].message.tool_calls[0]
958 +
959 + # Add the output messages from the first turn as input to the second turn
960 + for choice in response.choices:
961 + messages.append(choice.message.model_dump(exclude_none=True))
962 +
963 + # Add our tool output message
964 + messages.append(
965 + {
966 + "role": "tool",
967 + "tool_call_id": tool_call.id,
968 + "content": "20 degrees Celsius",
969 + },
970 + )
971 +
972 + # Test the Harmony messages for the second turn's input
973 + req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
974 + input_messages_2, _, _ = serving_chat._make_request_with_harmony(req_2)
975 + verify_harmony_messages(
976 + input_messages_2,
977 + [
978 + {"role": "system"},
979 + {"role": "developer"},
980 + {"role": "user"},
981 + {
982 + "role": "assistant",
983 + "channel": "commentary",
984 + "content": commentary_str,
985 + },
986 + {
987 + "role": "assistant",
988 + "channel": "commentary",
989 + "recipient": "functions.get_weather",
990 + "content": tool_args_str,
991 + },
992 + {
993 + "role": "tool",
994 + "author_name": "functions.get_weather",
995 + "channel": "commentary",
996 + "recipient": "assistant",
997 + "content": "20 degrees Celsius",
998 + },
999 + ],
1000 + )
1001 +
1002 + @pytest.mark.asyncio
1003 + async def test_tools_and_reasoning(
1004 + self, serving_chat, stream, weather_tools, weather_messages_start
1005 + ):
1006 + tools = weather_tools
1007 + messages = list(weather_messages_start)
1008 +
1009 + # Test the Harmony messages for the first turn's input
1010 + req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1011 + input_messages, _, _ = serving_chat._make_request_with_harmony(req)
1012 + verify_harmony_messages(
1013 + input_messages,
1014 + [
1015 + {"role": "system"},
1016 + {"role": "developer", "tool_definitions": ["get_weather"]},
1017 + {"role": "user", "content": messages[0]["content"]},
1018 + ],
1019 + )
1020 +
1021 + # Test the Chat Completion response for the first turn's output
1022 + reasoning_str = "I'll call get_weather."
1023 + tool_args_str = '{"location": "Paris"}'
1024 + response_str = (
1025 + f"<|channel|>analysis<|message|>{reasoning_str}<|end|>"
1026 + "<|start|>assistant to=functions.get_weather<|channel|>commentary"
1027 + f"<|constrain|>json<|message|>{tool_args_str}<|call|>"
1028 + )
1029 + response = await self.generate_response_from_harmony_str(
1030 + serving_chat, req, response_str, stream=stream
1031 + )
1032 + verify_chat_response(
1033 + response,
1034 + reasoning=reasoning_str,
1035 + tool_calls=[("get_weather", tool_args_str)],
1036 + )
1037 +
1038 + tool_call = response.choices[0].message.tool_calls[0]
1039 +
1040 + # Add the output messages from the first turn as input to the second turn
1041 + for choice in response.choices:
1042 + messages.append(choice.message.model_dump(exclude_none=True))
1043 +
1044 + # Add our tool output message
1045 + messages.append(
1046 + {
1047 + "role": "tool",
1048 + "tool_call_id": tool_call.id,
1049 + "content": "20 degrees Celsius",
1050 + },
1051 + )
1052 +
1053 + # Test the Harmony messages for the second turn's input
1054 + req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1055 + input_messages_2, _, _ = serving_chat._make_request_with_harmony(req_2)
1056 + verify_harmony_messages(
1057 + input_messages_2,
1058 + [
1059 + {"role": "system"},
1060 + {"role": "developer"},
1061 + {"role": "user"},
1062 + {
1063 + "role": "assistant",
1064 + "channel": "analysis",
1065 + "content": reasoning_str,
1066 + },
1067 + {
1068 + "role": "assistant",
1069 + "channel": "commentary",
1070 + "recipient": "functions.get_weather",
1071 + "content": tool_args_str,
1072 + },
1073 + {
1074 + "role": "tool",
1075 + "author_name": "functions.get_weather",
1076 + "channel": "commentary",
1077 + "recipient": "assistant",
1078 + "content": "20 degrees Celsius",
1079 + },
1080 + ],
1081 + )
1082 +
1083 + @pytest.mark.asyncio
1084 + async def test_multi_turn_tools_and_reasoning(
1085 + self, serving_chat, stream, weather_tools, weather_messages_start
1086 + ):
1087 + tools = weather_tools
1088 + messages = list(weather_messages_start)
1089 +
1090 + # Test the Harmony messages for the first turn's input
1091 + req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools)
1092 + input_messages, _, _ = serving_chat._make_request_with_harmony(req)
1093 + verify_harmony_messages(
1094 + input_messages,
1095 + [
1096 + {"role": "system"},
1097 + {"role": "developer", "tool_definitions": ["get_weather"]},
1098 + {"role": "user", "content": messages[0]["content"]},
1099 + ],
1100 + )
1101 +
1102 + # Test the Chat Completion response for the first turn's output
1103 + reasoning_str = "I'll call get_weather."
1104 + paris_tool_args_str = '{"location": "Paris"}'
1105 + response_str = (
1106 + f"<|channel|>analysis<|message|>{reasoning_str}<|end|>"
1107 + "<|start|>assistant to=functions.get_weather<|channel|>commentary"
1108 + f"<|constrain|>json<|message|>{paris_tool_args_str}<|call|>"
1109 + )
1110 + response = await self.generate_response_from_harmony_str(
1111 + serving_chat, req, response_str, stream=stream
1112 + )
1113 + verify_chat_response(
1114 + response,
1115 + reasoning=reasoning_str,
1116 + tool_calls=[("get_weather", paris_tool_args_str)],
1117 + )
1118 +
1119 + tool_call = response.choices[0].message.tool_calls[0]
1120 +
1121 + # Add the output messages from the first turn as input to the second turn
1122 + for choice in response.choices:
1123 + messages.append(choice.message.model_dump(exclude_none=True))
1124 +
1125 + # Add our tool output message
1126 + messages.append(
1127 + {
1128 + "role": "tool",
1129 + "tool_call_id": tool_call.id,
1130 + "content": "20 degrees Celsius",
1131 + },
1132 + )
1133 +
1134 + # Test the Harmony messages for the second turn's input
1135 + req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1136 + input_messages_2, _, _ = serving_chat._make_request_with_harmony(req_2)
1137 + verify_harmony_messages(
1138 + input_messages_2,
1139 + [
1140 + {"role": "system"},
1141 + {"role": "developer"},
1142 + {"role": "user"},
1143 + {
1144 + "role": "assistant",
1145 + "channel": "analysis",
1146 + "content": reasoning_str,
1147 + },
1148 + {
1149 + "role": "assistant",
1150 + "channel": "commentary",
1151 + "recipient": "functions.get_weather",
1152 + "content": paris_tool_args_str,
1153 + },
1154 + {
1155 + "role": "tool",
1156 + "author_name": "functions.get_weather",
1157 + "channel": "commentary",
1158 + "recipient": "assistant",
1159 + "content": "20 degrees Celsius",
1160 + },
1161 + ],
1162 + )
1163 +
1164 + # Test the Chat Completion response for the second turn's output
1165 + paris_weather_str = "The weather in Paris today is 20 degrees Celsius."
1166 + response_str = f"<|channel|>final<|message|>{paris_weather_str}<|end|>"
1167 + response_2 = await self.generate_response_from_harmony_str(
1168 + serving_chat, req_2, response_str, stream=stream
1169 + )
1170 + verify_chat_response(response_2, content=paris_weather_str)
1171 +
1172 + # Add the output messages from the second turn as input to the third turn
1173 + for choice in response_2.choices:
1174 + messages.append(choice.message.model_dump(exclude_none=True))
1175 +
1176 + # Add a new user message for the third turn
1177 + messages.append(
1178 + {
1179 + "role": "user",
1180 + "content": "What's the weather like in Boston today?",
1181 + },
1182 + )
1183 +
1184 + # Test the Harmony messages for the third turn's input
1185 + req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1186 + input_messages_3, _, _ = serving_chat._make_request_with_harmony(req_3)
1187 + verify_harmony_messages(
1188 + input_messages_3,
1189 + [
1190 + {"role": "system"},
1191 + {"role": "developer"},
1192 + {"role": "user"},
1193 + {
1194 + "role": "assistant",
1195 + "channel": "commentary",
1196 + "recipient": "functions.get_weather",
1197 + "content": paris_tool_args_str,
1198 + },
1199 + {
1200 + "role": "tool",
1201 + "author_name": "functions.get_weather",
1202 + "channel": "commentary",
1203 + "recipient": "assistant",
1204 + "content": "20 degrees Celsius",
1205 + },
1206 + {
1207 + "role": "assistant",
1208 + "channel": "final",
1209 + "content": paris_weather_str,
1210 + },
1211 + {"role": "user", "content": messages[-1]["content"]},
1212 + ],
1213 + )
1214 +
1215 + # Test the Chat Completion response for the third turn's output
1216 + reasoning_str = "I'll call get_weather."
1217 + boston_tool_args_str = '{"location": "Boston"}'
1218 + response_str = (
1219 + f"<|channel|>analysis<|message|>{reasoning_str}<|end|>"
1220 + "<|start|>assistant to=functions.get_weather<|channel|>commentary"
1221 + f"<|constrain|>json<|message|>{boston_tool_args_str}<|call|>"
1222 + )
1223 + response_3 = await self.generate_response_from_harmony_str(
1224 + serving_chat, req, response_str, stream=stream
1225 + )
1226 + verify_chat_response(
1227 + response_3,
1228 + reasoning=reasoning_str,
1229 + tool_calls=[("get_weather", boston_tool_args_str)],
1230 + )
1231 +
1232 + tool_call = response_3.choices[0].message.tool_calls[0]
1233 +
1234 + # Add the output messages from the third turn as input to the fourth turn
1235 + for choice in response_3.choices:
1236 + messages.append(choice.message.model_dump(exclude_none=True))
1237 +
1238 + # Add our tool output message
1239 + messages.append(
1240 + {
1241 + "role": "tool",
1242 + "tool_call_id": tool_call.id,
1243 + "content": "10 degrees Celsius",
1244 + },
1245 + )
1246 +
1247 + # Test the Harmony messages for the fourth turn's input
1248 + req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1249 + input_messages_4, _, _ = serving_chat._make_request_with_harmony(req_4)
1250 + verify_harmony_messages(
1251 + input_messages_4,
1252 + [
1253 + {"role": "system"},
1254 + {"role": "developer"},
1255 + {"role": "user"},
1256 + {"role": "assistant"},
1257 + {"role": "tool"},
1258 + {
1259 + "role": "assistant",
1260 + "channel": "final",
1261 + },
1262 + {"role": "user"},
1263 + {
1264 + "role": "assistant",
1265 + "channel": "analysis",
1266 + "content": reasoning_str,
1267 + },
1268 + {
1269 + "role": "assistant",
1270 + "channel": "commentary",
1271 + "recipient": "functions.get_weather",
1272 + "content": boston_tool_args_str,
1273 + },
1274 + {
1275 + "role": "tool",
1276 + "author_name": "functions.get_weather",
1277 + "channel": "commentary",
1278 + "recipient": "assistant",
1279 + "content": "10 degrees Celsius",
1280 + },
1281 + ],
1282 + )
1283 +
1284 + @pytest.mark.asyncio
1285 + async def test_non_tool_reasoning(self, serving_chat):
1286 + messages: list[dict[str, Any]] = [
1287 + {
1288 + "role": "user",
1289 + "content": "What's 2+2?",
1290 + },
1291 + {
1292 + "role": "assistant",
1293 + "reasoning": "Adding 2 and 2 is easy. The result is 4.",
1294 + "content": "4",
1295 + },
1296 + ]
1297 + req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1298 + input_messages, _, _ = serving_chat._make_request_with_harmony(req)
1299 +
1300 + verify_harmony_messages(
1301 + input_messages,
1302 + [
1303 + {"role": "system"},
1304 + {"role": "developer"},
1305 + {"role": "user", "content": messages[0]["content"]},
1306 + # The reasoning that would have resulted in an analysis message is
1307 + # dropped because of a later assistant message to the final channel.
1308 + {
1309 + "role": "assistant",
1310 + "channel": "final",
1311 + "content": messages[1]["content"],
1312 + },
1313 + ],
1314 + )
1315 +
1316 + @pytest.mark.asyncio
1317 + async def test_non_tool_reasoning_empty_content(self, serving_chat):
1318 + messages: list[dict[str, Any]] = [
1319 + {
1320 + "role": "user",
1321 + "content": "What's 2+2?",
1322 + },
1323 + {
1324 + "role": "assistant",
1325 + "reasoning": "Adding 2 and 2 is easy. The result is 4.",
1326 + "content": "",
1327 + },
1328 + ]
1329 + req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1330 + input_messages, _, _ = serving_chat._make_request_with_harmony(req)
1331 +
1332 + verify_harmony_messages(
1333 + input_messages,
1334 + [
1335 + {"role": "system"},
1336 + {"role": "developer"},
1337 + {"role": "user", "content": messages[0]["content"]},
1338 + {
1339 + "role": "assistant",
1340 + "channel": "analysis",
1341 + "content": messages[1]["reasoning"],
1342 + },
1343 + ],
1344 + )
1345 +
1346 + @pytest.mark.asyncio
1347 + async def test_non_tool_reasoning_empty_content_list(self, serving_chat):
1348 + messages: list[dict[str, Any]] = [
1349 + {
1350 + "role": "user",
1351 + "content": "What's 2+2?",
1352 + },
1353 + {
1354 + "role": "assistant",
1355 + "reasoning": "Adding 2 and 2 is easy. The result is 4.",
1356 + "content": [],
1357 + },
1358 + ]
1359 + req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1360 + input_messages, _, _ = serving_chat._make_request_with_harmony(req)
1361 +
1362 + verify_harmony_messages(
1363 + input_messages,
1364 + [
1365 + {"role": "system"},
1366 + {"role": "developer"},
1367 + {"role": "user", "content": messages[0]["content"]},
1368 + {
1369 + "role": "assistant",
1370 + "channel": "analysis",
1371 + "content": messages[1]["reasoning"],
1372 + },
1373 + ],
1374 + )
tests/entrypoints/openai/utils.py new
+190
@@ -0,0 +1,190 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import json
4 +from collections.abc import AsyncGenerator
5 +from typing import Any
6 +
7 +from vllm.entrypoints.openai.protocol import (
8 + ChatCompletionResponse,
9 + ChatCompletionResponseChoice,
10 + ChatCompletionStreamResponse,
11 + ChatMessage,
12 + UsageInfo,
13 +)
14 +
15 +
16 +async def accumulate_streaming_response(
17 + stream_generator: AsyncGenerator[str, None],
18 +) -> ChatCompletionResponse:
19 + """
20 + Accumulate streaming SSE chunks into a complete ChatCompletionResponse.
21 +
22 + This helper parses the SSE format and builds up the complete response
23 + by combining all the delta chunks.
24 + """
25 + accumulated_content = ""
26 + accumulated_reasoning = None
27 + accumulated_tool_calls: list[dict[str, Any]] = []
28 + role = None
29 + finish_reason = None
30 + response_id = None
31 + created = None
32 + model = None
33 + index = 0
34 +
35 + async for chunk_str in stream_generator:
36 + # Skip empty lines and [DONE] marker
37 + if not chunk_str.strip() or chunk_str.strip() == "data: [DONE]":
38 + continue
39 +
40 + # Parse SSE format: "data: {json}\n\n"
41 + if chunk_str.startswith("data: "):
42 + json_str = chunk_str[6:].strip()
43 + try:
44 + chunk_data = json.loads(json_str)
45 + # print(f"DEBUG: Parsed chunk_data: {chunk_data}")
46 + chunk = ChatCompletionStreamResponse(**chunk_data)
47 +
48 + # Store metadata from first chunk
49 + if response_id is None:
50 + response_id = chunk.id
51 + created = chunk.created
52 + model = chunk.model
53 +
54 + # Process each choice in the chunk
55 + for choice in chunk.choices:
56 + if choice.delta.role:
57 + role = choice.delta.role
58 + if choice.delta.content:
59 + accumulated_content += choice.delta.content
60 + if choice.delta.reasoning:
61 + if accumulated_reasoning is None:
62 + accumulated_reasoning = ""
63 + accumulated_reasoning += choice.delta.reasoning
64 + if choice.delta.tool_calls:
65 + # Accumulate tool calls
66 + for tool_call_delta in choice.delta.tool_calls:
67 + # Find or create the tool call at this index
68 + while len(accumulated_tool_calls) <= tool_call_delta.index:
69 + accumulated_tool_calls.append(
70 + {
71 + "id": None,
72 + "type": "function",
73 + "function": {"name": "", "arguments": ""},
74 + }
75 + )
76 +
77 + if tool_call_delta.id:
78 + accumulated_tool_calls[tool_call_delta.index]["id"] = (
79 + tool_call_delta.id
80 + )
81 + if tool_call_delta.function:
82 + if tool_call_delta.function.name:
83 + accumulated_tool_calls[tool_call_delta.index][
84 + "function"
85 + ]["name"] += tool_call_delta.function.name
86 + if tool_call_delta.function.arguments:
87 + accumulated_tool_calls[tool_call_delta.index][
88 + "function"
89 + ]["arguments"] += tool_call_delta.function.arguments
90 +
91 + if choice.finish_reason:
92 + finish_reason = choice.finish_reason
93 + if choice.index is not None:
94 + index = choice.index
95 +
96 + except json.JSONDecodeError:
97 + continue
98 +
99 + # Build the final message
100 + message_kwargs = {
101 + "role": role or "assistant",
102 + "content": accumulated_content if accumulated_content else None,
103 + "reasoning": accumulated_reasoning,
104 + }
105 +
106 + # Only include tool_calls if there are any
107 + if accumulated_tool_calls:
108 + message_kwargs["tool_calls"] = [
109 + {"id": tc["id"], "type": tc["type"], "function": tc["function"]}
110 + for tc in accumulated_tool_calls
111 + ]
112 +
113 + message = ChatMessage(**message_kwargs)
114 +
115 + # Build the final response
116 + choice = ChatCompletionResponseChoice(
117 + index=index,
118 + message=message,
119 + finish_reason=finish_reason or "stop",
120 + )
121 +
122 + # Create usage info (with dummy values for tests)
123 + usage = UsageInfo(
124 + prompt_tokens=0,
125 + completion_tokens=0,
126 + total_tokens=0,
127 + )
128 +
129 + response = ChatCompletionResponse(
130 + id=response_id or "chatcmpl-test",
131 + object="chat.completion",
132 + created=created or 0,
133 + model=model or "test-model",
134 + choices=[choice],
135 + usage=usage,
136 + )
137 +
138 + return response
139 +
140 +
141 +def verify_harmony_messages(
142 + messages: list[Any], expected_messages: list[dict[str, Any]]
143 +):
144 + assert len(messages) == len(expected_messages)
145 + for msg, expected in zip(messages, expected_messages):
146 + if "role" in expected:
147 + assert msg.author.role == expected["role"]
148 + if "author_name" in expected:
149 + assert msg.author.name == expected["author_name"]
150 + if "channel" in expected:
151 + assert msg.channel == expected["channel"]
152 + if "recipient" in expected:
153 + assert msg.recipient == expected["recipient"]
154 + if "content" in expected:
155 + assert msg.content[0].text == expected["content"]
156 + if "content_type" in expected:
157 + assert msg.content_type == expected["content_type"]
158 + if "tool_definitions" in expected:
159 + # Check that the tool definitions match the expected list of tool names
160 + actual_tools = [t.name for t in msg.content[0].tools["functions"].tools]
161 + assert actual_tools == expected["tool_definitions"]
162 +
163 +
164 +def verify_chat_response(
165 + response: ChatCompletionResponse,
166 + content: str | None = None,
167 + reasoning: str | None = None,
168 + tool_calls: list[tuple[str, str]] | None = None,
169 +):
170 + assert len(response.choices) == 1
171 + message = response.choices[0].message
172 +
173 + if content is not None:
174 + assert message.content == content
175 + else:
176 + assert not message.content
177 +
178 + if reasoning is not None:
179 + assert message.reasoning == reasoning
180 + else:
181 + assert not message.reasoning
182 +
183 + if tool_calls:
184 + assert message.tool_calls is not None
185 + assert len(message.tool_calls) == len(tool_calls)
186 + for tc, (expected_name, expected_args) in zip(message.tool_calls, tool_calls):
187 + assert tc.function.name == expected_name
188 + assert tc.function.arguments == expected_args
189 + else:
190 + assert not message.tool_calls
vllm/entrypoints/openai/parser/harmony_utils.py
+204 -21
@@ -232,7 +232,177 @@ def parse_response_input(
232 return msg
233
234
235 +def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]:
236 + """
237 + Parse a list of messages from request.messages in the Chat Completion API to
238 + Harmony messages.
239 + """
240 + msgs: list[Message] = []
241 + tool_id_names: dict[str, str] = {}
242 +
243 + # Collect tool id to name mappings for tool response recipient values
244 + for chat_msg in chat_msgs:
245 + for tool_call in chat_msg.get("tool_calls", []):
246 + tool_id_names[tool_call.get("id")] = tool_call.get("function", {}).get(
247 + "name"
248 + )
249 +
250 + for chat_msg in chat_msgs:
251 + msgs.extend(parse_chat_input_to_harmony_message(chat_msg, tool_id_names))
252 +
253 + msgs = auto_drop_analysis_messages(msgs)
254 + return msgs
255 +
256 +
257 +def auto_drop_analysis_messages(msgs: list[Message]) -> list[Message]:
258 + """
259 + Harmony models expect the analysis messages (representing raw chain of thought) to
260 + be dropped after an assistant message to the final channel is produced from the
261 + reasoning of those messages.
262 +
263 + The openai-harmony library does this if the very last assistant message is to the
264 + final channel, but it does not handle the case where we're in longer multi-turn
265 + conversations and the client gave us reasoning content from previous turns of
266 + the conversation with multiple assistant messages to the final channel in the
267 + conversation.
268 +
269 + So, we find the index of the last assistant message to the final channel and drop
270 + all analysis messages that precede it, leaving only the analysis messages that
271 + are relevant to the current part of the conversation.
272 + """
273 + last_assistant_final_index = -1
274 + for i in range(len(msgs) - 1, -1, -1):
275 + msg = msgs[i]
276 + if msg.author.role == "assistant" and msg.channel == "final":
277 + last_assistant_final_index = i
278 + break
279 +
280 + cleaned_msgs: list[Message] = []
281 + for i, msg in enumerate(msgs):
282 + if i < last_assistant_final_index and msg.channel == "analysis":
283 + continue
284 + cleaned_msgs.append(msg)
285 +
286 + return cleaned_msgs
287 +
288 +
289 +def flatten_chat_text_content(content: str | list | None) -> str | None:
290 + """
291 + Extract the text parts from a chat message content field and flatten them
292 + into a single string.
293 + """
294 + if isinstance(content, list):
295 + return "".join(
296 + item.get("text", "")
297 + for item in content
298 + if isinstance(item, dict) and item.get("type") == "text"
299 + )
300 + return content
301 +
302 +
303 +def parse_chat_input_to_harmony_message(
304 + chat_msg, tool_id_names: dict[str, str] | None = None
305 +) -> list[Message]:
306 + """
307 + Parse a message from request.messages in the Chat Completion API to
308 + Harmony messages.
309 + """
310 + tool_id_names = tool_id_names or {}
311 +
312 + if not isinstance(chat_msg, dict):
313 + # Handle Pydantic models
314 + chat_msg = chat_msg.model_dump(exclude_none=True)
315 +
316 + role = chat_msg.get("role")
317 + msgs: list[Message] = []
318 +
319 + # Assistant message with tool calls
320 + tool_calls = chat_msg.get("tool_calls", [])
321 +
322 + if role == "assistant" and tool_calls:
323 + content = flatten_chat_text_content(chat_msg.get("content"))
324 + if content:
325 + commentary_msg = Message.from_role_and_content(Role.ASSISTANT, content)
326 + commentary_msg = commentary_msg.with_channel("commentary")
327 + msgs.append(commentary_msg)
328 +
329 + reasoning_content = chat_msg.get("reasoning") or chat_msg.get(
330 + "reasoning_content"
331 + )
332 + if reasoning_content:
333 + analysis_msg = Message.from_role_and_content(
334 + Role.ASSISTANT, reasoning_content
335 + )
336 + analysis_msg = analysis_msg.with_channel("analysis")
337 + msgs.append(analysis_msg)
338 +
339 + for call in tool_calls:
340 + func = call.get("function", {})
341 + name = func.get("name", "")
342 + arguments = func.get("arguments", "") or ""
343 + msg = Message.from_role_and_content(Role.ASSISTANT, arguments)
344 + msg = msg.with_channel("commentary")
345 + msg = msg.with_recipient(f"functions.{name}")
346 + # Officially, this should be `<|constrain|>json` but there is not clear
347 + # evidence that improves accuracy over `json` and some anecdotes to the
348 + # contrary. Further testing of the different content_types is needed.
349 + msg = msg.with_content_type("json")
350 + msgs.append(msg)
351 + return msgs
352 +
353 + # Tool role message (tool output)
354 + if role == "tool":
355 + tool_call_id = chat_msg.get("tool_call_id", "")
356 + name = tool_id_names.get(tool_call_id, "")
357 + content = chat_msg.get("content", "") or ""
358 + content = flatten_chat_text_content(content)
359 +
360 + msg = (
361 + Message.from_author_and_content(
362 + Author.new(Role.TOOL, f"functions.{name}"), content
363 + )
364 + .with_channel("commentary")
365 + .with_recipient("assistant")
366 + )
367 + return [msg]
368 +
369 + # Non-tool reasoning content
370 + reasoning_content = chat_msg.get("reasoning") or chat_msg.get("reasoning_content")
371 + if role == "assistant" and reasoning_content:
372 + analysis_msg = Message.from_role_and_content(Role.ASSISTANT, reasoning_content)
373 + analysis_msg = analysis_msg.with_channel("analysis")
374 + msgs.append(analysis_msg)
375 +
376 + # Default: user/assistant/system messages with content
377 + content = chat_msg.get("content") or ""
378 + if content is None:
379 + content = ""
380 + if isinstance(content, str):
381 + contents = [TextContent(text=content)]
382 + else:
383 + # TODO: Support refusal.
384 + contents = [TextContent(text=c.get("text", "")) for c in content]
385 +
386 + # Only add assistant messages if they have content, as reasoning or tool calling
387 + # assistant messages were already added above.
388 + if role == "assistant" and contents and contents[0].text:
389 + msg = Message.from_role_and_contents(role, contents)
390 + # Send non-tool assistant messages to the final channel
391 + msg = msg.with_channel("final")
392 + msgs.append(msg)
393 + # For user/system/developer messages, add them directly even if no content.
394 + elif role != "assistant":
395 + msg = Message.from_role_and_contents(role, contents)
396 + msgs.append(msg)
397 +
398 + return msgs
399 +
400 +
401 def parse_input_to_harmony_message(chat_msg) -> list[Message]:
402 + """
403 + Parse a message from request.previous_input_messages in the Responsees API to
404 + Harmony messages.
405 + """
406 if not isinstance(chat_msg, dict):
407 # Handle Pydantic models
408 chat_msg = chat_msg.model_dump(exclude_none=True)
@@ -258,14 +428,7 @@ def parse_input_to_harmony_message(chat_msg) -> list[Message]:
428 if role == "tool":
429 name = chat_msg.get("name", "")
430 content = chat_msg.get("content", "") or ""
261 - if isinstance(content, list):
262 - # Handle array format for tool message content
263 - # by concatenating all text parts.
264 - content = "".join(
265 - item.get("text", "")
266 - for item in content
267 - if isinstance(item, dict) and item.get("type") == "text"
268 - )
431 + content = flatten_chat_text_content(content)
432
433 msg = Message.from_author_and_content(
434 Author.new(Role.TOOL, f"functions.{name}"), content
@@ -623,20 +786,40 @@ def parse_output_into_messages(token_ids: Iterable[int]) -> StreamableParser:
786 def parse_chat_output(
787 token_ids: Sequence[int],
788 ) -> tuple[str | None, str | None, bool]:
789 + """
790 + Parse the output of a Harmony chat completion into reasoning and final content.
791 + Note that when the `openai` tool parser is used, serving_chat only uses this
792 + for the reasoning content and gets the final content from the tool call parser.
793 +
794 + When the `openai` tool parser is not enabled, or when `GptOssReasoningParser` is
795 + in use,this needs to return the final content without any tool calls parsed.
796 +
797 + Empty reasoning or final content is returned as None instead of an empty string.
798 + """
799 parser = parse_output_into_messages(token_ids)
800 output_msgs = parser.messages
801 is_tool_call = False # TODO: update this when tool call is supported
629 - if len(output_msgs) == 0:
630 - # The generation has stopped during reasoning.
631 - reasoning = parser.current_content
632 - final_content = None
633 - elif len(output_msgs) == 1:
634 - # The generation has stopped during final message.
635 - reasoning = output_msgs[0].content[0].text
636 - final_content = parser.current_content
637 - else:
638 - reasoning_msg = output_msgs[:-1]
639 - final_msg = output_msgs[-1]
640 - reasoning = "\n".join([msg.content[0].text for msg in reasoning_msg])
641 - final_content = final_msg.content[0].text
802 +
803 + # Get completed messages from the parser
804 + reasoning_texts = [
805 + msg.content[0].text for msg in output_msgs if msg.channel == "analysis"
806 + ]
807 + final_texts = [
808 + msg.content[0].text for msg in output_msgs if msg.channel != "analysis"
809 + ]
810 +
811 + # Extract partial messages from the parser
812 + if parser.current_channel == "analysis" and parser.current_content:
813 + reasoning_texts.append(parser.current_content)
814 + elif parser.current_channel != "analysis" and parser.current_content:
815 + final_texts.append(parser.current_content)
816 +
817 + # Flatten multiple messages into a single string
818 + reasoning: str | None = "\n".join(reasoning_texts)
819 + final_content: str | None = "\n".join(final_texts)
820 +
821 + # Return None instead of empty string since existing callers check for None
822 + reasoning = reasoning or None
823 + final_content = final_content or None
824 +
825 return reasoning, final_content, is_tool_call
vllm/entrypoints/openai/serving_chat.py
+10 -3
@@ -27,8 +27,8 @@ from vllm.entrypoints.openai.parser.harmony_utils import (
27 get_stop_tokens_for_assistant_actions,
28 get_streamable_parser_for_assistant,
29 get_system_message,
30 + parse_chat_inputs_to_harmony_messages,
31 parse_chat_output,
31 - parse_input_to_harmony_message,
32 render_for_completion,
33 )
34 from vllm.entrypoints.openai.protocol import (
@@ -822,6 +822,9 @@ class OpenAIServingChat(OpenAIServing):
822
823 if delta_message is not None:
824 harmony_tools_streamed[i] = True
825 + elif cur_channel == "commentary":
826 + # Tool call preambles meant to be shown to the user
827 + delta_message = DeltaMessage(content=delta_text)
828 else:
829 delta_message = None
830 # handle streaming deltas for tools with named tool_choice
@@ -1770,6 +1773,11 @@ class OpenAIServingChat(OpenAIServing):
1773 ):
1774 messages: list[OpenAIMessage] = []
1775
1776 + # because of issues with pydantic we need to potentially
1777 + # re-serialize the tool_calls field of the request
1778 + # for more info: see comment in `maybe_serialize_tool_calls`
1779 + maybe_serialize_tool_calls(request)
1780 +
1781 # Add system message.
1782 # NOTE: In Chat Completion API, browsing is enabled by default
1783 # if the model supports it. TODO: Support browsing.
@@ -1788,8 +1796,7 @@ class OpenAIServingChat(OpenAIServing):
1796 messages.append(dev_msg)
1797
1798 # Add user message.
1791 - for chat_msg in request.messages:
1792 - messages.extend(parse_input_to_harmony_message(chat_msg))
1799 + messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
1800
1801 # Render prompt token ids.
1802 prompt_token_ids = render_for_completion(messages)
vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py
+6 -1
@@ -43,6 +43,7 @@ class OpenAIToolParser(ToolParser):
43 parser = parse_output_into_messages(token_ids)
44 tool_calls = []
45 final_content = None
46 + commentary_content = None
47
48 if len(parser.messages) > 0:
49 for msg in parser.messages:
@@ -75,11 +76,15 @@ class OpenAIToolParser(ToolParser):
76 )
77 elif msg.channel == "final":
78 final_content = msg_text
79 + elif msg.channel == "commentary" and not msg.recipient:
80 + commentary_content = msg_text
81
82 return ExtractedToolCallInformation(
83 tools_called=len(tool_calls) > 0,
84 tool_calls=tool_calls,
82 - content=final_content,
85 + # prefer final content over commentary content if both are present
86 + # commentary content is tool call preambles meant to be shown to the user
87 + content=final_content or commentary_content,
88 )
89
90 def extract_tool_calls_streaming(