[Bugfix] GPT-OSS Autodrop reasoning in Response API and cleanup (#45048)
Signed-off-by: Yifan Zong <yzong@redhat.com> Co-authored-by: Ben Browning <bbrownin@redhat.com>
yzong-rh committed
Jun 23, 2026 at 09:36 UTC
f59db63732e5c4f72e64d524cecc87e6fe0c4453
5 files changed
+137
-46
tests/entrypoints/openai/chat_completion/test_serving_chat.py
+28
-10
@@ -1429,15 +1429,23 @@ class TestServingChatWithHarmony:
1429
input_messages_2, _ = serving_chat.online_renderer._make_request_with_harmony(
1430
req_2
1431
)
1432
+ expected_input_messages_2 = [
1433
+ {"role": "system"},
1434
+ {"role": "user"},
1435
+ ]
1436
+ if include_reasoning:
1437
+ expected_input_messages_2.append(
1438
+ {
1439
+ "role": "assistant",
1440
+ "channel": "analysis",
1441
+ }
1442
+ )
1443
+ expected_input_messages_2.append(
1444
+ {"role": "assistant", "channel": "final", "content": final_str}
1445
+ )
1446
verify_harmony_messages(
1447
input_messages_2,
1434
- [
1435
- {"role": "system"},
1436
- {"role": "user"},
1437
- # The analysis message should be dropped on subsequent inputs because
1438
- # of the subsequent assistant message to the final channel.
1439
- {"role": "assistant", "channel": "final", "content": final_str},
1440
- ],
1448
+ expected_input_messages_2,
1449
)
1450
1451
@pytest.mark.asyncio
@@ -1635,7 +1643,6 @@ class TestServingChatWithHarmony:
1643
{
1644
"role": "assistant",
1645
"channel": "analysis",
1638
- "content": reasoning_str,
1646
},
1647
{
1648
"role": "assistant",
@@ -1684,6 +1691,11 @@ class TestServingChatWithHarmony:
1691
{"role": "system"},
1692
{"role": "developer"},
1693
{"role": "user"},
1694
+ {
1695
+ "role": "assistant",
1696
+ "channel": "analysis",
1697
+ "content": reasoning_str,
1698
+ },
1699
{
1700
"role": "assistant",
1701
"channel": "commentary",
@@ -1749,6 +1761,10 @@ class TestServingChatWithHarmony:
1761
{"role": "system"},
1762
{"role": "developer"},
1763
{"role": "user"},
1764
+ {
1765
+ "role": "assistant",
1766
+ "channel": "analysis",
1767
+ },
1768
{"role": "assistant"},
1769
{"role": "tool"},
1770
{
@@ -1798,8 +1814,10 @@ class TestServingChatWithHarmony:
1814
[
1815
{"role": "system"},
1816
{"role": "user", "content": messages[0]["content"]},
1801
- # The reasoning that would have resulted in an analysis message is
1802
- # dropped because of a later assistant message to the final channel.
1817
+ {
1818
+ "role": "assistant",
1819
+ "channel": "analysis",
1820
+ },
1821
{
1822
"role": "assistant",
1823
"channel": "final",
tests/entrypoints/openai/parser/test_harmony_render_parity.py
+103
-9
@@ -23,11 +23,15 @@ from openai.types.responses import ResponseFunctionToolCall
23
24
from tests.entrypoints.openai.utils import verify_harmony_messages
25
from vllm.entrypoints.openai.parser.harmony_utils import (
26
+ get_encoding,
27
get_system_message,
28
parse_chat_input_to_harmony_message,
29
render_for_completion,
30
)
30
-from vllm.entrypoints.openai.responses.harmony import response_input_to_harmony
31
+from vllm.entrypoints.openai.responses.harmony import (
32
+ response_input_to_harmony,
33
+ response_previous_input_to_harmony,
34
+)
35
36
# Use a fixed date so the system message is deterministic across both paths.
37
_DATE = "2025-01-01"
@@ -395,6 +399,9 @@ class TestResponseInputToHarmonyRenderParity:
399
reasoning trace. Reasoning traces in between commentary-channel tool
400
calls must survive as analysis-channel messages in both paths.
401
"""
402
+ first_reasoning = "I need current weather first."
403
+ second_reasoning = "Now I need the weekly forecast."
404
+
405
prev_call_1 = ResponseFunctionToolCall(
406
id="fc_1",
407
call_id="call_1",
@@ -420,7 +427,7 @@ class TestResponseInputToHarmonyRenderParity:
427
chat_msgs += parse_chat_input_to_harmony_message(
428
{
429
"role": "assistant",
423
- "reasoning": "I need current weather first.",
430
+ "reasoning": first_reasoning,
431
"tool_calls": [
432
{
433
"id": "call_1",
@@ -440,7 +447,7 @@ class TestResponseInputToHarmonyRenderParity:
447
chat_msgs += parse_chat_input_to_harmony_message(
448
{
449
"role": "assistant",
443
- "reasoning": "Now I need the weekly forecast.",
450
+ "reasoning": second_reasoning,
451
"tool_calls": [
452
{
453
"id": "call_2",
@@ -472,9 +479,7 @@ class TestResponseInputToHarmonyRenderParity:
479
# First reasoning + tool call
480
{
481
"type": "reasoning",
475
- "content": [
476
- {"type": "reasoning_text", "text": "I need current weather first."}
477
- ],
482
+ "content": [{"type": "reasoning_text", "text": first_reasoning}],
483
},
484
{
485
"type": "function_call",
@@ -492,7 +497,7 @@ class TestResponseInputToHarmonyRenderParity:
497
"content": [
498
{
499
"type": "reasoning_text",
495
- "text": "Now I need the weekly forecast.",
500
+ "text": second_reasoning,
501
}
502
],
503
},
@@ -512,6 +517,95 @@ class TestResponseInputToHarmonyRenderParity:
517
for item in resp_input
518
]
519
515
- assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
516
- [_system()] + resp_msgs
520
+ chat_completion_tokens = render_for_completion([_system()] + chat_msgs)
521
+ responses_tokens = render_for_completion([_system()] + resp_msgs)
522
+
523
+ assert chat_completion_tokens == responses_tokens
524
+
525
+ rendered_prompt = get_encoding().decode(chat_completion_tokens)
526
+ assert first_reasoning in rendered_prompt
527
+ assert second_reasoning in rendered_prompt
528
+
529
+ def test_completed_turns_drop_reasoning(self):
530
+ """Validates that reasoning from completed turns is dropped, while
531
+ reasoning from the current in-progress tool-call turn is preserved
532
+ in both chat completions and responses previous_input_messages."""
533
+ first_turn_reasoning = "FIRST_TURN_REASONING"
534
+ second_turn_reasoning = "SECOND_TURN_REASONING"
535
+
536
+ chat_completion_msgs = []
537
+ for chat_message in [
538
+ {"role": "user", "content": "What is 2+2?"},
539
+ {
540
+ "role": "assistant",
541
+ "reasoning": first_turn_reasoning,
542
+ "content": "The answer is 4.",
543
+ },
544
+ {"role": "user", "content": "Now what is 3+3?"},
545
+ {
546
+ "role": "assistant",
547
+ "reasoning": second_turn_reasoning,
548
+ "tool_calls": [
549
+ {
550
+ "id": "call_1",
551
+ "function": {
552
+ "name": "calc",
553
+ "arguments": '{"a":3,"b":3}',
554
+ },
555
+ }
556
+ ],
557
+ },
558
+ ]:
559
+ chat_completion_msgs.extend(
560
+ parse_chat_input_to_harmony_message(chat_message)
561
+ )
562
+
563
+ responses_prev_input_msgs = []
564
+ for responses_message in [
565
+ {
566
+ "author": {"role": "user"},
567
+ "content": [{"type": "text", "text": "What is 2+2?"}],
568
+ },
569
+ {
570
+ "author": {"role": "assistant"},
571
+ "channel": "analysis",
572
+ "content": [{"type": "text", "text": first_turn_reasoning}],
573
+ },
574
+ {
575
+ "author": {"role": "assistant"},
576
+ "channel": "final",
577
+ "content": [{"type": "text", "text": "The answer is 4."}],
578
+ },
579
+ {
580
+ "author": {"role": "user"},
581
+ "content": [{"type": "text", "text": "Now what is 3+3?"}],
582
+ },
583
+ {
584
+ "author": {"role": "assistant"},
585
+ "channel": "analysis",
586
+ "content": [{"type": "text", "text": second_turn_reasoning}],
587
+ },
588
+ {
589
+ "author": {"role": "assistant"},
590
+ "channel": "commentary",
591
+ "recipient": "functions.calc",
592
+ "content_type": "json",
593
+ "content": [{"type": "text", "text": '{"a":3,"b":3}'}],
594
+ },
595
+ ]:
596
+ responses_prev_input_msgs.extend(
597
+ response_previous_input_to_harmony(responses_message)
598
+ )
599
+
600
+ chat_completion_tokens = render_for_completion(
601
+ [_system()] + chat_completion_msgs
602
+ )
603
+ responses_tokens = render_for_completion(
604
+ [_system()] + responses_prev_input_msgs
605
)
606
+
607
+ assert chat_completion_tokens == responses_tokens
608
+
609
+ rendered_prompt = get_encoding().decode(responses_tokens)
610
+ assert first_turn_reasoning not in rendered_prompt
611
+ assert second_turn_reasoning in rendered_prompt
vllm/entrypoints/openai/parser/harmony_utils.py
+5
-2
@@ -13,6 +13,7 @@ from openai_harmony import (
13
HarmonyEncodingName,
14
Message,
15
ReasoningEffort,
16
+ RenderConversationConfig,
17
Role,
18
StreamableParser,
19
SystemContent,
@@ -220,7 +221,6 @@ def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]:
221
for chat_msg in chat_msgs:
222
msgs.extend(parse_chat_input_to_harmony_message(chat_msg, tool_id_names))
223
223
- msgs = auto_drop_analysis_messages(msgs)
224
return msgs
225
226
@@ -447,9 +447,12 @@ def parse_chat_input_to_harmony_message(
447
448
449
def render_for_completion(messages: list[Message]) -> list[int]:
450
+ messages = auto_drop_analysis_messages(messages)
451
conversation = Conversation.from_messages(messages)
452
token_ids = get_encoding().render_conversation_for_completion(
452
- conversation, Role.ASSISTANT
453
+ conversation,
454
+ Role.ASSISTANT,
455
+ config=RenderConversationConfig(auto_drop_analysis=False),
456
)
457
return token_ids
458
vllm/entrypoints/openai/responses/harmony.py
+1
-1
@@ -67,7 +67,7 @@ def _parse_harmony_format_message(chat_msg: dict) -> Message:
67
contents = [TextContent(text="")]
68
69
if name:
70
- msg = Message.from_author_and_contents(Author.new(Role(role), name), contents)
70
+ msg = Message(author=Author.new(Role(role), name), content=contents)
71
else:
72
msg = Message.from_role_and_contents(Role(role), contents)
73
vllm/entrypoints/openai/responses/serving.py
-24
@@ -1191,30 +1191,6 @@ class OpenAIServingResponses(OpenAIServing):
1191
# instructions are ignored.
1192
prev_msgs = self.msg_store[prev_response.id]
1193
1194
- # FIXME(woosuk): The slice-delete-reappend cycle below is
1195
- # currently a no-op --- it removes messages then puts them all
1196
- # back unfiltered. It may be intentionally deferred (see FIXME
1197
- # above) or redundant if the Harmony encoder already strips
1198
- # analysis messages at render time. If analysis messages need
1199
- # to be dropped here, add a channel != "analysis" filter when
1200
- # re-appending, similar to auto_drop_analysis_messages in
1201
- # harmony_utils.py.
1202
- if len(prev_msgs) > 0:
1203
- last_msg = prev_msgs[-1]
1204
- assert isinstance(last_msg, OpenAIHarmonyMessage)
1205
- if last_msg.channel == "final":
1206
- prev_final_msg_idx = -1
1207
- for i in range(len(prev_msgs) - 2, -1, -1):
1208
- prev_msg_i = prev_msgs[i]
1209
- assert isinstance(prev_msg_i, OpenAIHarmonyMessage)
1210
- if prev_msg_i.channel == "final":
1211
- prev_final_msg_idx = i
1212
- break
1213
- recent_turn_msgs = prev_msgs[prev_final_msg_idx + 1 :]
1214
- del prev_msgs[prev_final_msg_idx + 1 :]
1215
- for msg in recent_turn_msgs:
1216
- assert isinstance(msg, OpenAIHarmonyMessage)
1217
- prev_msgs.append(msg)
1194
messages.extend(prev_msgs)
1195
# Append the new input.
1196
# Responses API supports simple text inputs without chat format.