@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 0cd6f767e3

[Bugfix][Frontend][gpt-oss] Recover raw tail when Harmony parser ends non-terminal (#47379)

yzong-rh committed Jul 4, 2026 at 10:46 UTC 0cd6f767e36f78598c2a932478b59c2f78897ff3

4 files changed +145 -83

tests/entrypoints/unit_tests/test_context.py

+22 -13

@@ -74,7 +74,7 @@ class FakeHarmonyParser(HarmonyParser):

74 self.reasoning_parser = None

75 self.tool_parser = None

76 self._chunk_results: list[ChunkResult] = []

77 - self._flush_results: list[Segment | None] = []

77 + self._flush_results: list[list[Segment]] = []

78 self.processed_chunks: list[list[int]] = []

80 def enqueue_chunk_result(

@@ -89,7 +89,7 @@ class FakeHarmonyParser(HarmonyParser):

89 )

90 )

92 - def enqueue_flush_result(self, segment: Segment | None) -> None:

92 + def enqueue_flush_result(self, segment: list[Segment]) -> None:

93 self._flush_results.append(segment)

95 def process_chunk(self, token_ids) -> ChunkResult:

@@ -98,10 +98,10 @@ class FakeHarmonyParser(HarmonyParser):

98 return self._chunk_results.pop(0)

99 return ChunkResult(segments=[], reasoning_token_count=0)

100

101 - def flush(self) -> Segment | None:

101 + def flush(self) -> list[Segment]:

102 if self._flush_results:

103 return self._flush_results.pop(0)

104 - return None

104 + return []

105

106

107 def make_harmony_context(

@@ -598,13 +598,21 @@ async def test_streaming_message_synchronization():

598 content=[TextContent(text=response_text)],

599 recipient=Role.USER,

600 )

601 - flush_segment = Segment(

602 - channel="commentary",

603 - recipient=None,

604 - delta="",

605 - completed_message=message,

606 - )

607 - parser.enqueue_flush_result(flush_segment)

601 + flush_segments = [

602 + Segment(

603 + channel="final",

604 + recipient=None,

605 + delta=response_text,

606 + completed_message=None,

607 + ),

608 + Segment(

609 + channel="final",

610 + recipient=None,

611 + delta="",

612 + completed_message=message,

613 + ),

614 + ]

615 + parser.enqueue_flush_result(flush_segments)

616

617 # Create another output to trigger synchronization via flush()

618 context.append_output(

@@ -618,8 +626,9 @@ async def test_streaming_message_synchronization():

626 assert context.num_init_messages == 1

627 assert context._messages[2].content[0].text == response_text

628 assert context.last_append_flush_status is True

621 - assert len(context.last_append_segments) == 1

622 - assert context.last_append_segments[0].completed_message is message

629 + assert len(context.last_append_segments) == 2

630 + assert context.last_append_segments[-2].delta == response_text

631 + assert context.last_append_segments[-1].completed_message is message

632

633

634 def test_turn_metrics_copy_and_reset():

tests/parser/test_harmony.py

+70 -34

@@ -7,7 +7,6 @@ from collections.abc import Sequence

7 import pytest

8 from openai_harmony import (

9 Conversation,

10 - HarmonyError,

10 Message,

11 RenderConversationConfig,

12 Role,

@@ -51,6 +50,15 @@ def chat_request():

50 )

53 +@pytest.fixture

54 +def malformed_msgs_str() -> list[str]:

55 + return [

56 + "<|channel|>analysis<|message|>thinking<|end|>",

58 + '<|start|>assistant<|channel|>final {"answer": "hi"}<|return|>',

59 + ]

60 +

61 +

62 def encode_output(harmony_str: str) -> list[int]:

63 return get_encoding().encode(harmony_str, allowed_special="all")

@@ -131,13 +139,22 @@ def tool_call_entries(delta_message) -> list[tuple[int, str | None, str | None]]

139 ]

140

141

142 +def assert_parser_is_reset(harmony_parser: HarmonyParser):

143 + assert harmony_parser._parser is None

144 + assert harmony_parser._num_processed_messages == 0

145 + assert harmony_parser._current_message_tokens == []

146 +

147 +

148 class TestFlush:

149 def test_flush(self, harmony_parser):

150 harmony_parser.process_chunk(

151 encode_output("<|channel|>analysis<|message|>Think")

152 )

153

140 - flushed = harmony_parser.flush()

154 + flushed_segments = harmony_parser.flush()

155 + assert flushed_segments is not None

156 + assert len(flushed_segments) == 1

157 + flushed = flushed_segments[0]

158

159 assert flushed is not None

160 assert flushed.channel == "analysis"

@@ -145,15 +162,27 @@ class TestFlush:

162 assert flushed.delta == ""

163 assert flushed.completed_message is not None

164 assert get_text(flushed.completed_message) == "Think"

148 - assert harmony_parser._parser is None

165 + assert_parser_is_reset(harmony_parser)

166

150 - def test_flush_raises_and_resets_on_non_terminal_eos(self, harmony_parser):

151 - harmony_parser.process_chunk(encode_output("<|channel|>analysis"))

167 + def test_flush_recovers_invalid_output(self, harmony_parser, malformed_msgs_str):

168 + for msg_str in malformed_msgs_str[:-1]:

169 + chunk = harmony_parser.process_chunk(encode_output(msg_str))

170 + assert "".join(segment.delta for segment in chunk.segments) == "thinking"

171

153 - with pytest.raises(HarmonyError):

154 - harmony_parser.flush()

172 + last_msg_str = malformed_msgs_str[-1]

173 + harmony_parser.process_chunk(encode_output(last_msg_str))

174 + flushed_segments = harmony_parser.flush()

175 + assert len(flushed_segments) == 2

176 + delta_segment = flushed_segments[0]

177 + message_segment = flushed_segments[1]

178

156 - assert harmony_parser._parser is None

179 + assert delta_segment.channel == "final"

180 + assert delta_segment.recipient is None

181 + assert delta_segment.delta == last_msg_str

182 + assert message_segment.channel == "final"

183 + assert message_segment.recipient is None

184 + assert get_text(message_segment.completed_message) == last_msg_str

185 + assert_parser_is_reset(harmony_parser)

186

187

188 class TestParse:

@@ -364,7 +393,7 @@ class TestParse:

393 assert reasoning is None

394 assert content == "I'm in the middle of answering"

395 assert tool_calls is None

367 - assert harmony_parser._parser is None

396 + assert_parser_is_reset(harmony_parser)

397

398 def test_interrupted_reasoning_first_message(self, harmony_parser, chat_request):

399 reasoning, content, tool_calls = harmony_parser.parse(

@@ -378,7 +407,7 @@ class TestParse:

407 assert reasoning == "I'm in the middle of thinking"

408 assert content is None

409 assert tool_calls is None

381 - assert harmony_parser._parser is None

410 + assert_parser_is_reset(harmony_parser)

411

412 def test_truncated_output(self, harmony_parser, chat_request):

413 reasoning, content, tool_calls = harmony_parser.parse(

@@ -394,24 +423,23 @@ class TestParse:

423 assert reasoning == "I'm thinking."

424 assert content == "I'm in the middle of answering"

425 assert tool_calls is None

397 - assert harmony_parser._parser is None

426 + assert_parser_is_reset(harmony_parser)

427

399 - def test_malformed_final_recovers_raw_content(self, harmony_parser, chat_request):

400 - raw_output = (

401 - "<|channel|>analysis<|message|>thinking<|end|>"

402 - '<|start|>assistant<|channel|>final {"answer": "hi"}<|return|>'

403 - )

428 + def test_malformed_msgs_recovers_raw_content(

429 + self, harmony_parser, chat_request, malformed_msgs_str

430 + ):

431 + combined_output = "".join(malformed_msgs_str)

432

433 reasoning, content, tool_calls = harmony_parser.parse(

406 - raw_output,

434 + "",

435 chat_request,

408 - model_output_token_ids=encode_output(raw_output),

436 + model_output_token_ids=encode_output(combined_output),

437 )

438

411 - assert content == raw_output

412 - assert reasoning is None

439 + assert reasoning == "thinking"

440 + assert content == "thinking\n" + malformed_msgs_str[-1]

441 assert tool_calls is None

414 - assert harmony_parser._parser is None

442 + assert_parser_is_reset(harmony_parser)

443

444 @pytest.mark.parametrize(

445 ("harmony_str", "expected_content"),

@@ -489,7 +517,7 @@ class TestParseDelta:

517 assert second_delta is not None

518 assert second_delta.content == "Answer"

519 assert second_delta.reasoning is None

492 - assert parser._parser is None

520 + assert_parser_is_reset(parser)

521

522 def test_multi_token(self, gpt_oss_tokenizer, chat_request):

523 parser = HarmonyParser(gpt_oss_tokenizer)

@@ -506,25 +534,33 @@ class TestParseDelta:

534 assert delta.reasoning is None

535 assert not delta.tool_calls

536

509 - def test_malformed_final_recovers_raw_content(

510 - self, gpt_oss_tokenizer, chat_request

537 + def test_malformed_msgs_recovers_raw_content(

538 + self, gpt_oss_tokenizer, chat_request, malformed_msgs_str

539 ):

540 parser = HarmonyParser(gpt_oss_tokenizer)

541

514 - delta = parser.parse_delta(

515 - delta_text='final {"answer": "hi"}',

516 - delta_token_ids=encode_output(

517 - '<|channel|>final {"answer": "hi"}<|return|>'

518 - ),

542 + for msg_str in malformed_msgs_str[:-1]:

543 + delta = parser.parse_delta(

544 + delta_text="",

545 + delta_token_ids=encode_output(msg_str),

546 + request=chat_request,

547 + finished=False,

548 + )

549 + assert delta.reasoning or delta.content == "thinking"

550 + assert not delta.tool_calls

551 +

552 + last_delta = parser.parse_delta(

553 + delta_text="",

554 + delta_token_ids=encode_output(malformed_msgs_str[-1]),

555 request=chat_request,

556 finished=True,

557 )

558

523 - assert delta is not None

524 - assert delta.content == 'final {"answer": "hi"}'

525 - assert delta.reasoning is None

526 - assert not delta.tool_calls

527 - assert parser._parser is None

559 + assert last_delta is not None

560 + assert last_delta.content == malformed_msgs_str[-1]

561 + assert last_delta.reasoning is None

562 + assert not last_delta.tool_calls

563 + assert_parser_is_reset(parser)

564

565 @pytest.mark.parametrize("tool_channel", ["commentary", "analysis"])

566 def test_tool_call_split_across_deltas(

vllm/entrypoints/openai/responses/context.py

+6 -6

@@ -18,7 +18,7 @@ from openai.types.responses.response_output_item import McpCall

18 from openai.types.responses.response_output_message import ResponseOutputMessage

19 from openai.types.responses.response_output_text import ResponseOutputText

20 from openai.types.responses.tool import Mcp

21 -from openai_harmony import Author, HarmonyError, Message, Role, TextContent

21 +from openai_harmony import Author, Message, Role, TextContent

23 from vllm import envs

24 from vllm.entrypoints.chat_utils import (

@@ -616,7 +616,7 @@ class HarmonyContext(ConversationContext):

616 self.num_tool_output_tokens = 0

617

618 self.last_append_segments: list[Segment] = []

619 - self.last_append_flush_status: bool | HarmonyError = False

619 + self.last_append_flush_status: bool = False

620

621 # Turn tracking - replaces multiple individual tracking variables

622 self.current_turn_metrics = TurnMetrics()

@@ -643,10 +643,10 @@ class HarmonyContext(ConversationContext):

643

644 if output.finished:

645 self.finish_reason = output.outputs[0].finish_reason

646 - flushed = self.response_parser.flush()

647 - if flushed is not None:

648 - segments.append(flushed)

649 - self.last_append_flush_status = flushed is not None

646 + flushed_segments = self.response_parser.flush()

647 + if flushed_segments:

648 + segments.extend(flushed_segments)

649 + self.last_append_flush_status = len(flushed_segments) > 0

650 self.all_turn_metrics.append(self.current_turn_metrics.copy())

651 self.current_turn_metrics.reset()

652

vllm/parser/harmony.py

+47 -30

@@ -9,7 +9,7 @@ from dataclasses import dataclass

9 from enum import Enum, auto

10 from typing import TYPE_CHECKING, NamedTuple

12 -from openai_harmony import HarmonyError

12 +from openai_harmony import HarmonyError, Message, Role

14 from vllm.entrypoints.chat_utils import make_tool_call_id

15 from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest

@@ -91,6 +91,9 @@ class HarmonyParser(DelegatingParser):

91 self._next_tool_call_index = 0

92 self._num_processed_messages = 0

94 + # For error recovery

95 + self._current_message_tokens: list[int] = []

96 +

97 @property

98 def _harmony_parser(self) -> StreamableParser:

99 """Lazily initializes the Harmony parser."""

@@ -107,32 +110,48 @@ class HarmonyParser(DelegatingParser):

110 self._num_processed_messages += 1

111 return msg

112

110 - def flush(self) -> Segment | None:

113 + def flush(self) -> list[Segment]:

114 + segments: list[Segment] = []

115 try:

116 self._harmony_parser.process_eos()

117 msg = self._poll_completed_message()

118 except HarmonyError:

119 logger.warning(

120 "Harmony parser ended in a non-terminal state; returning the "

117 - "raw unparsed output. This usually indicates a malformed "

118 - "assistant turn, e.g. a 'final' channel missing the "

119 - "<|message|> delimiter."

121 + "recovered raw output."

122 )

121 - raise

122 - finally:

123 - # Reset to the initial assistant-parser state for the next turn.

124 - self._parser = None

125 - self._num_processed_messages = 0

123

127 - if msg is None:

128 - return None

124 + final_channel = "final"

125 + text = self.model_tokenizer.decode(self._current_message_tokens)

126 + segments.append(

127 + Segment(

128 + channel=final_channel,

129 + recipient=None,

130 + delta=text,

131 + completed_message=None,

132 + )

133 + )

134 + msg = Message.from_role_and_content(Role.ASSISTANT, text).with_channel(

135 + final_channel

136 + )

137 +

138 + # Reset to the initial assistant-parser state for the next turn.

139 + self._parser = None

140 + self._num_processed_messages = 0

141 + self._current_message_tokens.clear()

142

130 - return Segment(

131 - channel=msg.channel,

132 - recipient=msg.recipient,

133 - delta="",

134 - completed_message=msg,

143 + if msg is None:

144 + return segments

145 +

146 + segments.append(

147 + Segment(

148 + channel=msg.channel,

149 + recipient=msg.recipient,

150 + delta="",

151 + completed_message=msg,

152 + )

153 )

154 + return segments

155

156 def parse(

157 self,

@@ -147,12 +166,9 @@ class HarmonyParser(DelegatingParser):

166 Callers must decide whether to surface them.

167 """

168 result = self.process_chunk(model_output_token_ids)

150 - try:

151 - flushed_segment = self.flush()

152 - except HarmonyError:

153 - return None, model_output, None

154 - if flushed_segment is not None:

155 - result.segments.append(flushed_segment)

169 + flushed_segments = self.flush()

170 + if flushed_segments:

171 + result.segments.extend(flushed_segments)

172

173 reasoning_parts: list[str] = []

174 content_parts: list[str] = []

@@ -209,13 +225,9 @@ class HarmonyParser(DelegatingParser):

225 )

226 result = self.process_chunk(delta_token_ids)

227 if finished:

212 - try:

213 - flushed_segment = self.flush()

214 - except HarmonyError:

215 - self._next_tool_call_index = 0

216 - return DeltaMessage(content=delta_text)

217 - if flushed_segment is not None:

218 - result.segments.append(flushed_segment)

228 + flushed_segments = self.flush()

229 + if flushed_segments:

230 + result.segments.extend(flushed_segments)

231 combined_content = ""

232 combined_reasoning = ""

233 tool_messages: list[DeltaToolCall] = []

@@ -298,6 +310,11 @@ class HarmonyParser(DelegatingParser):

310 delta = self._harmony_parser.last_content_delta or ""

311 completed_message = self._poll_completed_message()

312

313 + if completed_message is not None:

314 + self._current_message_tokens.clear()

315 + else:

316 + self._current_message_tokens.append(token_id)

317 +

318 if channel == "analysis" or (

319 channel == "commentary" and recipient is not None

320 ):