A high-throughput and memory-efficient inference and serving engine for LLMs
[Bugfix] Chat Completions Harmony Refactor Clean up (#45464)
Signed-off-by: Yifan Zong <yzong@redhat.com> Co-authored-by: Ben Browning <bbrownin@redhat.com>
yzong-rh committed
Jun 15, 2026 at 14:45 UTC
51ec5cf08f4e3e6f55f51edfbbc29c645f1c4dcd
3 files changed
+46
-38
tests/parser/test_harmony.py
+24
-24
@@ -118,12 +118,17 @@ def tool_call_payloads(delta_message) -> list:
118
]
119
120
121
-def combined_tool_arguments(delta_message) -> dict[int, str]:
122
- combined: dict[int, str] = {}
123
- for tool_call in tool_call_payloads(delta_message):
124
- combined.setdefault(tool_call.index, "")
125
- combined[tool_call.index] += tool_call.function.arguments
126
- return combined
121
+def tool_call_entries(delta_message) -> list[tuple[int, str | None, str | None]]:
122
+ if delta_message is None or not delta_message.tool_calls:
123
+ return []
124
+ return [
125
+ (
126
+ tool_call.index,
127
+ tool_call.function.name if tool_call.function else None,
128
+ tool_call.function.arguments if tool_call.function else None,
129
+ )
130
+ for tool_call in delta_message.tool_calls
131
+ ]
132
133
134
class TestParse:
@@ -481,18 +486,14 @@ class TestParseDelta:
486
assert first_delta is not None
487
assert first_delta.reasoning == "Thinking"
488
assert first_delta.content is None
484
- assert [tool.function.name for tool in tool_call_headers(first_delta)] == [
485
- "get_weather"
489
+ assert tool_call_entries(first_delta) == [
490
+ (0, "get_weather", '{"location": '),
491
]
487
- assert combined_tool_arguments(first_delta) == {0: '{"location": '}
488
- assert {tool.index for tool in first_delta.tool_calls} == {0}
492
493
assert second_delta is not None
494
assert second_delta.reasoning is None
495
assert second_delta.content is None
493
- assert not tool_call_headers(second_delta)
494
- assert combined_tool_arguments(second_delta) == {0: '"Paris"}'}
495
- assert {tool.index for tool in second_delta.tool_calls} == {0}
496
+ assert tool_call_entries(second_delta) == [(0, None, '"Paris"}')]
497
498
def test_commentary_preamble_streaming(self, gpt_oss_tokenizer, chat_request):
499
parser = HarmonyParser(gpt_oss_tokenizer)
@@ -601,8 +602,7 @@ class TestParseDelta:
602
assert delta is not None
603
assert delta.reasoning == "Reasoning about query..."
604
assert delta.content == "Done"
604
- assert [tool.function.name for tool in tool_call_headers(delta)] == ["search"]
605
- assert combined_tool_arguments(delta) == {0: '{"query": "vllm"}'}
605
+ assert tool_call_entries(delta) == [(0, "search", '{"query": "vllm"}')]
606
607
def test_tool_index_across_calls(self, gpt_oss_tokenizer, chat_request):
608
parser = HarmonyParser(gpt_oss_tokenizer)
@@ -665,22 +665,22 @@ class TestParseDelta:
665
finished=False,
666
)
667
668
+ assert tool_call_entries(first_delta) == [
669
+ (0, "tool_a", '{"a": 1}'),
670
+ (1, "tool_b", '{"b": '),
671
+ ]
672
assert [tool.index for tool in tool_call_headers(first_delta)] == [0, 1]
669
- assert combined_tool_arguments(first_delta) == {
670
- 0: '{"a": 1}',
671
- 1: '{"b": ',
672
- }
673
674
assert second_delta is not None
675
+ assert tool_call_entries(second_delta) == [(1, None, "2")]
676
assert [tool.index for tool in tool_call_payloads(second_delta)] == [1]
676
- assert combined_tool_arguments(second_delta) == {1: "2"}
677
678
assert third_delta is not None
679
assert third_delta.content == "Done"
680
- assert combined_tool_arguments(third_delta) == {
681
- 1: "}",
682
- 2: '{"c": 3}',
683
- }
680
+ assert tool_call_entries(third_delta) == [
681
+ (1, None, "}"),
682
+ (2, "tool_c", '{"c": 3}'),
683
+ ]
684
assert [tool.index for tool in tool_call_headers(third_delta)] == [2]
685
686
vllm/entrypoints/serve/render/serving.py
+2
-1
@@ -189,17 +189,18 @@ class OpenAIServingRender:
189
self.trust_request_chat_template = trust_request_chat_template
190
self.enable_auto_tools = enable_auto_tools
191
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
192
+ self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
193
self.parser: type[Parser] | None = ParserManager.get_parser(
194
tool_parser_name=tool_parser,
195
reasoning_parser_name=reasoning_parser,
196
enable_auto_tools=enable_auto_tools,
197
model_name=model_config.model,
198
+ is_harmony=self.use_harmony,
199
)
200
self.default_chat_template_kwargs: dict[str, Any] = (
201
default_chat_template_kwargs or {}
202
)
203
self.log_error_stack = log_error_stack
202
- self.use_harmony = model_config.hf_config.model_type == "gpt_oss"
204
self.supports_browsing = False
205
self.supports_code_interpreter = False
206
vllm/parser/harmony.py
+20
-13
@@ -68,18 +68,18 @@ class HarmonyParser(DelegatingParser):
68
def __init__(self, tokenizer, tools=None, *args, **kwargs):
69
super().__init__(tokenizer, tools, *args, **kwargs)
70
71
- if self._reasoning_parser and not isinstance(
72
- self._reasoning_parser, GptOssReasoningParser
71
+ if self.reasoning_parser and not isinstance(
72
+ self.reasoning_parser, GptOssReasoningParser
73
):
74
raise ValueError(
75
"Harmony requires GptOssReasoningParser, "
76
- f"got {self._reasoning_parser.__class__.__name__}."
76
+ f"got {self.reasoning_parser.__class__.__name__}."
77
)
78
79
- if self._tool_parser and not isinstance(self._tool_parser, GptOssToolParser):
79
+ if self.tool_parser and not isinstance(self.tool_parser, GptOssToolParser):
80
raise ValueError(
81
"Harmony requires GptOssToolParser, "
82
- f"got {self._tool_parser.__class__.__name__}."
82
+ f"got {self.tool_parser.__class__.__name__}."
83
)
84
85
self._harmony_parser = get_streamable_parser_for_assistant()
@@ -209,11 +209,11 @@ class HarmonyParser(DelegatingParser):
209
segment.channel, segment.recipient
210
)
211
match segment_type:
212
- case _SegmentType.REASONING:
212
+ case _SegmentType.REASONING if self.reasoning_parser:
213
combined_reasoning += segment.delta
214
case _SegmentType.CONTENT:
215
combined_content += segment.delta
216
- case _SegmentType.TOOL:
216
+ case _SegmentType.TOOL if self.tool_parser:
217
assert segment.recipient is not None
218
if prev_recipient != segment.recipient:
219
tool_name = extract_function_from_recipient(segment.recipient)
@@ -233,13 +233,20 @@ class HarmonyParser(DelegatingParser):
233
self._next_tool_call_index += 1
234
prev_recipient = segment.recipient
235
elif segment.delta:
236
- tool_call_index = self._next_tool_call_index - 1
237
- tool_messages.append(
238
- DeltaToolCall(
239
- index=tool_call_index,
240
- function=DeltaFunctionCall(arguments=segment.delta),
236
+ idx = self._next_tool_call_index - 1
237
+ if tool_messages:
238
+ tool_msg = tool_messages[-1]
239
+ assert tool_msg.index == idx
240
+ fn = tool_msg.function
241
+ assert fn is not None and fn.arguments is not None
242
+ fn.arguments += segment.delta
243
+ else:
244
+ tool_messages.append(
245
+ DeltaToolCall(
246
+ index=idx,
247
+ function=DeltaFunctionCall(arguments=segment.delta),
248
+ )
249
)
242
- )
250
251
if not combined_content and not combined_reasoning and not tool_messages:
252
return None