A high-throughput and memory-efficient inference and serving engine for LLMs
[Refactor] Chat Completions Streaming Harmony Refactor and Bugfixes (#45104)
Signed-off-by: Yifan Zong <yzong@redhat.com>
yzong-rh committed
Jun 11, 2026 at 21:09 UTC
e0871ad2259768add6dc43e2972bd364d0d13086
5 files changed
+394
-733
tests/entrypoints/openai/chat_completion/test_serving_chat_stream_harmony.py
deleted
-471
@@ -1,471 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""
4
-Unit tests for harmony streaming delta extraction.
5
-"""
6
-
7
-from dataclasses import dataclass, field
8
-from unittest.mock import patch
9
-
10
-import pytest
11
-
12
-from vllm.entrypoints.openai.chat_completion.stream_harmony import (
13
- TokenState,
14
- extract_harmony_streaming_delta,
15
-)
16
-
17
-
18
-@dataclass
19
-class MockMessage:
20
- """Mock message object for testing."""
21
-
22
- channel: str | None = None
23
- recipient: str | None = None
24
-
25
-
26
-@dataclass
27
-class MockStreamableParser:
28
- """Mock StreamableParser for testing without openai_harmony dependency."""
29
-
30
- messages: list[MockMessage] = field(default_factory=list)
31
-
32
-
33
-class TestExtractHarmonyStreamingDelta:
34
- """Tests for extract_harmony_streaming_delta function."""
35
-
36
- @pytest.mark.parametrize(
37
- "delta_text,expected_content",
38
- [
39
- ("Hello, world!", "Hello, world!"),
40
- ("", ""),
41
- ],
42
- )
43
- def test_final_channel_returns_content_delta(self, delta_text, expected_content):
44
- """Test that final channel returns a DeltaMessage with content."""
45
- parser = MockStreamableParser()
46
-
47
- # Updated to use TokenState list
48
- token_states = [TokenState(channel="final", recipient=None, text=delta_text)]
49
-
50
- delta_message, tools_streamed = extract_harmony_streaming_delta(
51
- harmony_parser=parser,
52
- token_states=token_states,
53
- prev_recipient=None,
54
- include_reasoning=False,
55
- )
56
-
57
- assert delta_message is not None
58
- assert delta_message.content == expected_content
59
- assert tools_streamed is False
60
-
61
- @pytest.mark.parametrize(
62
- "include_reasoning,expected_has_message",
63
- [
64
- (True, True),
65
- (False, False),
66
- ],
67
- )
68
- def test_analysis_channel_reasoning(self, include_reasoning, expected_has_message):
69
- """Test analysis channel respects include_reasoning flag."""
70
- parser = MockStreamableParser()
71
- text = "Let me think..."
72
- token_states = [TokenState(channel="analysis", recipient=None, text=text)]
73
-
74
- delta_message, tools_streamed = extract_harmony_streaming_delta(
75
- harmony_parser=parser,
76
- token_states=token_states,
77
- prev_recipient=None,
78
- include_reasoning=include_reasoning,
79
- )
80
-
81
- if expected_has_message:
82
- assert delta_message is not None
83
- assert delta_message.reasoning == text
84
- else:
85
- assert delta_message is None
86
- assert tools_streamed is False
87
-
88
- @pytest.mark.parametrize("channel", ["commentary", "analysis"])
89
- @patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
90
- def test_new_tool_call(self, mock_make_tool_call_id, channel):
91
- """Test new tool call creation when recipient changes."""
92
- mock_make_tool_call_id.return_value = "call_test123"
93
- parser = MockStreamableParser()
94
-
95
- token_states = [
96
- TokenState(channel=channel, recipient="functions.get_weather", text="")
97
- ]
98
-
99
- delta_message, tools_streamed = extract_harmony_streaming_delta(
100
- harmony_parser=parser,
101
- token_states=token_states,
102
- prev_recipient=None,
103
- include_reasoning=False,
104
- )
105
-
106
- assert delta_message is not None
107
- assert len(delta_message.tool_calls) == 1
108
- tool_call = delta_message.tool_calls[0]
109
- assert tool_call.id == "call_test123"
110
- assert tool_call.type == "function"
111
- assert tool_call.function.name == "get_weather"
112
- assert tool_call.function.arguments == ""
113
- assert tool_call.index == 0
114
- assert tools_streamed is True
115
-
116
- @pytest.mark.parametrize("channel", ["commentary", "analysis"])
117
- def test_tool_call_argument_streaming(self, channel):
118
- """Test streaming tool call arguments (same recipient)."""
119
- parser = MockStreamableParser()
120
- args_text = '{"location": "Paris"}'
121
-
122
- token_states = [
123
- TokenState(
124
- channel=channel,
125
- recipient="functions.get_weather",
126
- text=args_text,
127
- )
128
- ]
129
-
130
- delta_message, tools_streamed = extract_harmony_streaming_delta(
131
- harmony_parser=parser,
132
- token_states=token_states,
133
- prev_recipient="functions.get_weather",
134
- include_reasoning=False,
135
- )
136
-
137
- assert delta_message is not None
138
- tool_call = delta_message.tool_calls[0]
139
- assert tool_call.id is None
140
- assert tool_call.function.arguments == args_text
141
- assert tool_call.index == 0
142
- assert tools_streamed is True
143
-
144
- @pytest.mark.parametrize("channel", ["commentary", "analysis"])
145
- def test_tool_call_empty_arguments_returns_none(self, channel):
146
- """Test empty delta_text with same recipient returns None."""
147
- parser = MockStreamableParser()
148
-
149
- token_states = [
150
- TokenState(channel=channel, recipient="functions.get_weather", text="")
151
- ]
152
-
153
- delta_message, tools_streamed = extract_harmony_streaming_delta(
154
- harmony_parser=parser,
155
- token_states=token_states,
156
- prev_recipient="functions.get_weather",
157
- include_reasoning=False,
158
- )
159
-
160
- assert delta_message is None
161
- assert tools_streamed is False
162
-
163
- def test_tool_call_index_from_previous_messages(self):
164
- """Test tool call index accounts for previous function messages."""
165
- messages = [
166
- MockMessage(channel="analysis", recipient=None), # Not counted
167
- MockMessage(channel="commentary", recipient="functions.tool1"), # Counted
168
- MockMessage(channel="final", recipient=None), # Not counted
169
- ]
170
- parser = MockStreamableParser(messages=messages)
171
-
172
- token_states = [
173
- TokenState(
174
- channel="commentary",
175
- recipient="functions.tool2",
176
- text="args",
177
- )
178
- ]
179
-
180
- delta_message, _ = extract_harmony_streaming_delta(
181
- harmony_parser=parser,
182
- token_states=token_states,
183
- prev_recipient="functions.tool2",
184
- include_reasoning=False,
185
- )
186
-
187
- assert delta_message.tool_calls[0].index == 1
188
-
189
- def test_returns_preambles_as_content(self):
190
- """Test that commentary with no recipient (preamble) is user content."""
191
- parser = MockStreamableParser()
192
- delta_text = "some text"
193
-
194
- token_states = [
195
- TokenState(channel="commentary", recipient=None, text=delta_text)
196
- ]
197
-
198
- delta_message, tools_streamed = extract_harmony_streaming_delta(
199
- harmony_parser=parser,
200
- token_states=token_states,
201
- prev_recipient=None,
202
- include_reasoning=True,
203
- )
204
-
205
- assert delta_message.content == delta_text
206
- assert tools_streamed is False
207
-
208
- @pytest.mark.parametrize("channel", ["commentary", "analysis"])
209
- @patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
210
- def test_new_tool_call_dotted_function_name(self, mock_make_tool_call_id, channel):
211
- mock_make_tool_call_id.return_value = "call_dotted123"
212
- parser = MockStreamableParser()
213
-
214
- token_states = [TokenState(channel=channel, recipient="math.sum", text="")]
215
-
216
- delta_message, tools_streamed = extract_harmony_streaming_delta(
217
- harmony_parser=parser,
218
- token_states=token_states,
219
- prev_recipient=None,
220
- include_reasoning=False,
221
- )
222
-
223
- assert delta_message is not None
224
- assert len(delta_message.tool_calls) == 1
225
- tool_call = delta_message.tool_calls[0]
226
- assert tool_call.id == "call_dotted123"
227
- assert tool_call.type == "function"
228
- assert tool_call.function.name == "math.sum"
229
- assert tool_call.function.arguments == ""
230
- assert tool_call.index == 0
231
- assert tools_streamed is True
232
-
233
- @pytest.mark.parametrize(
234
- "channel,recipient",
235
- [
236
- (None, None),
237
- ("unknown_channel", None),
238
- ("commentary", "browser.search"),
239
- ("commentary", "assistant"),
240
- ],
241
- )
242
- def test_returns_none_for_invalid_inputs(self, channel, recipient):
243
- """Test that invalid channel/recipient combinations return None."""
244
- parser = MockStreamableParser()
245
-
246
- token_states = [
247
- TokenState(channel=channel, recipient=recipient, text="some text")
248
- ]
249
-
250
- delta_message, tools_streamed = extract_harmony_streaming_delta(
251
- harmony_parser=parser,
252
- token_states=token_states,
253
- prev_recipient=None,
254
- include_reasoning=True,
255
- )
256
-
257
- assert delta_message is None
258
- assert tools_streamed is False
259
-
260
- def test_consecutive_token_grouping(self):
261
- """
262
- Test that consecutive tokens with the same channel/recipient
263
- are merged into a single processing group.
264
- """
265
- parser = MockStreamableParser()
266
- token_states = [
267
- TokenState("final", None, "H"),
268
- TokenState("final", None, "el"),
269
- TokenState("final", None, "lo"),
270
- TokenState("final", None, ","),
271
- TokenState("final", None, " World"),
272
- ]
273
-
274
- delta_message, _ = extract_harmony_streaming_delta(
275
- harmony_parser=parser,
276
- token_states=token_states,
277
- prev_recipient=None,
278
- include_reasoning=False,
279
- )
280
-
281
- assert delta_message is not None
282
- assert delta_message.content == "Hello, World"
283
-
284
- @patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
285
- def test_complex_batch_permutation(self, mock_make_id):
286
- """
287
- Test a complex permutation: Reasoning -> Tool Call -> Content.
288
- This verifies that multiple distinct actions in one batch
289
- are all captured in the single DeltaMessage.
290
- """
291
- mock_make_id.return_value = "call_batch_test"
292
- parser = MockStreamableParser()
293
-
294
- token_states = [
295
- # 1. Reasoning
296
- TokenState("analysis", None, "Reasoning about query..."),
297
- # 2. Tool Calling
298
- TokenState("commentary", "functions.search", '{"query":'),
299
- TokenState("commentary", "functions.search", ' "vllm"}'),
300
- # 3. Final Content
301
- TokenState("final", None, "."),
302
- ]
303
-
304
- delta_message, tools_streamed = extract_harmony_streaming_delta(
305
- harmony_parser=parser,
306
- token_states=token_states,
307
- prev_recipient=None,
308
- include_reasoning=True,
309
- )
310
-
311
- assert delta_message is not None
312
-
313
- assert delta_message.reasoning == "Reasoning about query..."
314
-
315
- # We expect 2 objects for 1 logical tool call:
316
- # 1. The definition (id, name, type)
317
- # 2. The arguments payload
318
- assert len(delta_message.tool_calls) == 2
319
-
320
- header = delta_message.tool_calls[0]
321
- payload = delta_message.tool_calls[1]
322
-
323
- assert header.function.name == "search"
324
- assert header.id == "call_batch_test"
325
- assert header.index == 0
326
-
327
- assert payload.index == 0
328
- assert payload.function.arguments == '{"query": "vllm"}'
329
-
330
- assert delta_message.content == "."
331
- assert tools_streamed is True
332
-
333
- @patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
334
- def test_tool_call_index_consistency_with_ongoing_call(self, mock_make_id):
335
- """
336
- Test that an ongoing tool call continuation and subsequent new calls
337
- maintain correct indexing when interleaved with content.
338
- """
339
- mock_make_id.side_effect = ["id_b", "id_c"]
340
-
341
- messages = [
342
- MockMessage(channel="commentary", recipient="functions.previous_tool")
343
- ]
344
- parser = MockStreamableParser(messages=messages)
345
-
346
- token_states = [
347
- TokenState("commentary", "functions.tool_a", '{"key_a": "val_a"}'),
348
- TokenState("final", None, "Thinking..."),
349
- TokenState("commentary", "functions.tool_b", '{"key_b": "val_b"}'),
350
- TokenState("final", None, " Thinking again..."),
351
- TokenState("commentary", "functions.tool_c", '{"key_c": "val_c"}'),
352
- ]
353
-
354
- delta_message, _ = extract_harmony_streaming_delta(
355
- harmony_parser=parser,
356
- token_states=token_states,
357
- prev_recipient="functions.tool_a",
358
- include_reasoning=False,
359
- )
360
-
361
- assert delta_message is not None
362
-
363
- tool_a_deltas = [t for t in delta_message.tool_calls if t.index == 1]
364
- assert len(tool_a_deltas) > 0
365
- assert tool_a_deltas[0].id is None
366
- assert tool_a_deltas[0].function.arguments == '{"key_a": "val_a"}'
367
-
368
- tool_b_header = next(t for t in delta_message.tool_calls if t.id == "id_b")
369
- assert tool_b_header.index == 2
370
- tool_b_args = next(
371
- t for t in delta_message.tool_calls if t.index == 2 and t.id is None
372
- )
373
- assert tool_b_args.function.arguments == '{"key_b": "val_b"}'
374
-
375
- tool_c_start = next(t for t in delta_message.tool_calls if t.id == "id_c")
376
- assert tool_c_start.index == 3
377
- tool_c_args = next(
378
- t for t in delta_message.tool_calls if t.index == 3 and t.id is None
379
- )
380
- assert tool_c_args.function.arguments == '{"key_c": "val_c"}'
381
-
382
- assert delta_message.content == "Thinking... Thinking again..."
383
-
384
-
385
-class TestToolCallsOnNonStandardChannels:
386
- """Tool calls are detected by recipient, not channel.
387
-
388
- Models sometimes emit tool calls on unexpected channels (e.g. ``comment``
389
- instead of ``commentary``). These tests verify that the streaming delta
390
- extraction is channel-agnostic for tool call detection.
391
- """
392
-
393
- @patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
394
- def test_prefixed_tool_call_on_comment_channel(self, mock_make_tool_call_id):
395
- mock_make_tool_call_id.return_value = "call_comment_chan"
396
- parser = MockStreamableParser()
397
-
398
- token_states = [
399
- TokenState(channel="comment", recipient="functions.get_weather", text="")
400
- ]
401
-
402
- delta_message, tools_streamed = extract_harmony_streaming_delta(
403
- harmony_parser=parser,
404
- token_states=token_states,
405
- prev_recipient=None,
406
- include_reasoning=False,
407
- )
408
-
409
- assert delta_message is not None
410
- assert len(delta_message.tool_calls) == 1
411
- assert delta_message.tool_calls[0].function.name == "get_weather"
412
- assert tools_streamed is True
413
-
414
- @patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
415
- def test_bare_tool_call_on_comment_channel(self, mock_make_tool_call_id):
416
- mock_make_tool_call_id.return_value = "call_bare_comment"
417
- parser = MockStreamableParser()
418
-
419
- token_states = [TokenState(channel="comment", recipient="get_weather", text="")]
420
-
421
- delta_message, tools_streamed = extract_harmony_streaming_delta(
422
- harmony_parser=parser,
423
- token_states=token_states,
424
- prev_recipient=None,
425
- include_reasoning=False,
426
- )
427
-
428
- assert delta_message is not None
429
- assert len(delta_message.tool_calls) == 1
430
- assert delta_message.tool_calls[0].function.name == "get_weather"
431
- assert tools_streamed is True
432
-
433
- def test_tool_call_arguments_on_comment_channel(self):
434
- parser = MockStreamableParser()
435
- args_text = '{"location": "Paris"}'
436
-
437
- token_states = [
438
- TokenState(
439
- channel="comment", recipient="functions.get_weather", text=args_text
440
- )
441
- ]
442
-
443
- delta_message, tools_streamed = extract_harmony_streaming_delta(
444
- harmony_parser=parser,
445
- token_states=token_states,
446
- prev_recipient="functions.get_weather",
447
- include_reasoning=False,
448
- )
449
-
450
- assert delta_message is not None
451
- assert delta_message.tool_calls[0].function.arguments == args_text
452
- assert tools_streamed is True
453
-
454
- def test_base_index_counts_tool_calls_on_comment_channel(self):
455
- messages = [
456
- MockMessage(channel="comment", recipient="functions.tool1"),
457
- ]
458
- parser = MockStreamableParser(messages=messages)
459
-
460
- token_states = [
461
- TokenState(channel="commentary", recipient="functions.tool2", text="args")
462
- ]
463
-
464
- delta_message, _ = extract_harmony_streaming_delta(
465
- harmony_parser=parser,
466
- token_states=token_states,
467
- prev_recipient="functions.tool2",
468
- include_reasoning=False,
469
- )
470
-
471
- assert delta_message.tool_calls[0].index == 1
tests/parser/test_harmony.py
+309
-25
@@ -94,16 +94,36 @@ def get_text(msg: Message) -> str:
94
return msg.content[0].text if msg.content else ""
95
96
97
-def visible_segments(result) -> list[tuple[str | None, str | None, str]]:
97
+def tool_call_tuples(tool_calls: list[FunctionCall] | None) -> list[tuple[str, str]]:
98
+ return [] if tool_calls is None else [(tc.name, tc.arguments) for tc in tool_calls]
99
+
100
+
101
+def tool_call_headers(delta_message) -> list:
102
+ if delta_message is None or not delta_message.tool_calls:
103
+ return []
104
return [
99
- (segment.channel, segment.recipient, segment.delta)
100
- for segment in result.segments
101
- if not segment.is_boundary and segment.delta
105
+ tool_call
106
+ for tool_call in delta_message.tool_calls
107
+ if tool_call.function and tool_call.function.name
108
]
109
110
105
-def tool_call_tuples(tool_calls: list[FunctionCall] | None) -> list[tuple[str, str]]:
106
- return [] if tool_calls is None else [(tc.name, tc.arguments) for tc in tool_calls]
111
+def tool_call_payloads(delta_message) -> list:
112
+ if delta_message is None or not delta_message.tool_calls:
113
+ return []
114
+ return [
115
+ tool_call
116
+ for tool_call in delta_message.tool_calls
117
+ if tool_call.function and tool_call.function.arguments
118
+ ]
119
+
120
+
121
+def combined_tool_arguments(delta_message) -> dict[int, str]:
122
+ combined: dict[int, str] = {}
123
+ for tool_call in tool_call_payloads(delta_message):
124
+ combined.setdefault(tool_call.index, "")
125
+ combined[tool_call.index] += tool_call.function.arguments
126
+ return combined
127
128
129
class TestParse:
@@ -394,6 +414,276 @@ class TestParse:
414
]
415
416
417
+class TestParseDelta:
418
+ def test_basic(self, gpt_oss_tokenizer, chat_request):
419
+ parser = HarmonyParser(gpt_oss_tokenizer)
420
+
421
+ first_delta = parser.parse_delta(
422
+ delta_text="",
423
+ delta_token_ids=encode_output("<|channel|>analysis<|message|>Thinking"),
424
+ request=chat_request,
425
+ finished=False,
426
+ )
427
+ second_delta = parser.parse_delta(
428
+ delta_text="",
429
+ delta_token_ids=encode_output(
430
+ "<|end|><|start|>assistant<|channel|>final<|message|>Answer"
431
+ ),
432
+ request=chat_request,
433
+ finished=False,
434
+ )
435
+
436
+ assert first_delta is not None
437
+ assert first_delta.reasoning == "Thinking"
438
+ assert first_delta.content is None
439
+ assert second_delta is not None
440
+ assert second_delta.content == "Answer"
441
+ assert second_delta.reasoning is None
442
+
443
+ def test_multi_token(self, gpt_oss_tokenizer, chat_request):
444
+ parser = HarmonyParser(gpt_oss_tokenizer)
445
+
446
+ delta = parser.parse_delta(
447
+ delta_text="",
448
+ delta_token_ids=encode_output("<|channel|>final<|message|>Hello, world!"),
449
+ request=chat_request,
450
+ finished=False,
451
+ )
452
+
453
+ assert delta is not None
454
+ assert delta.content == "Hello, world!"
455
+ assert delta.reasoning is None
456
+ assert not delta.tool_calls
457
+
458
+ @pytest.mark.parametrize("tool_channel", ["commentary", "analysis"])
459
+ def test_tool_call_split_across_deltas(
460
+ self, gpt_oss_tokenizer, chat_request, tool_channel
461
+ ):
462
+ parser = HarmonyParser(gpt_oss_tokenizer)
463
+
464
+ first_delta = parser.parse_delta(
465
+ delta_text="",
466
+ delta_token_ids=encode_output(
467
+ "<|channel|>analysis<|message|>Thinking<|end|>"
468
+ f"<|start|>assistant to=functions.get_weather<|channel|>{tool_channel}"
469
+ '<|constrain|>json<|message|>{"location": '
470
+ ),
471
+ request=chat_request,
472
+ finished=False,
473
+ )
474
+ second_delta = parser.parse_delta(
475
+ delta_text="",
476
+ delta_token_ids=encode_output('"Paris"}<|call|>'),
477
+ request=chat_request,
478
+ finished=False,
479
+ )
480
+
481
+ assert first_delta is not None
482
+ assert first_delta.reasoning == "Thinking"
483
+ assert first_delta.content is None
484
+ assert [tool.function.name for tool in tool_call_headers(first_delta)] == [
485
+ "get_weather"
486
+ ]
487
+ assert combined_tool_arguments(first_delta) == {0: '{"location": '}
488
+ assert {tool.index for tool in first_delta.tool_calls} == {0}
489
+
490
+ assert second_delta is not None
491
+ assert second_delta.reasoning is None
492
+ assert second_delta.content is None
493
+ assert not tool_call_headers(second_delta)
494
+ assert combined_tool_arguments(second_delta) == {0: '"Paris"}'}
495
+ assert {tool.index for tool in second_delta.tool_calls} == {0}
496
+
497
+ def test_commentary_preamble_streaming(self, gpt_oss_tokenizer, chat_request):
498
+ parser = HarmonyParser(gpt_oss_tokenizer)
499
+
500
+ delta = parser.parse_delta(
501
+ delta_text="",
502
+ delta_token_ids=encode_output(
503
+ "<|channel|>commentary<|message|>I'll search for that"
504
+ ),
505
+ request=chat_request,
506
+ finished=False,
507
+ )
508
+
509
+ assert delta is not None
510
+ assert delta.content == "I'll search for that"
511
+ assert delta.reasoning is None
512
+ assert not delta.tool_calls
513
+
514
+ def test_multiple_choices(self, gpt_oss_tokenizer, chat_request):
515
+ parser_a = HarmonyParser(gpt_oss_tokenizer)
516
+ parser_b = HarmonyParser(gpt_oss_tokenizer)
517
+
518
+ delta_a = parser_a.parse_delta(
519
+ delta_text="",
520
+ delta_token_ids=encode_output(
521
+ "<|channel|>analysis<|message|>Check weather<|end|>"
522
+ "<|start|>assistant to=functions.get_weather<|channel|>commentary"
523
+ '<|constrain|>json<|message|>{"location": "Paris"}'
524
+ ),
525
+ request=chat_request,
526
+ finished=False,
527
+ )
528
+ delta_b = parser_b.parse_delta(
529
+ delta_text="",
530
+ delta_token_ids=encode_output(
531
+ "<|channel|>analysis<|message|>Check time<|end|>"
532
+ "<|start|>assistant to=functions.get_time<|channel|>commentary"
533
+ '<|constrain|>json<|message|>{"timezone": "UTC"}'
534
+ ),
535
+ request=chat_request,
536
+ finished=False,
537
+ )
538
+
539
+ assert [tool.function.name for tool in tool_call_headers(delta_a)] == [
540
+ "get_weather"
541
+ ]
542
+ assert [tool.function.name for tool in tool_call_headers(delta_b)] == [
543
+ "get_time"
544
+ ]
545
+ assert {tool.index for tool in delta_a.tool_calls} == {0}
546
+ assert {tool.index for tool in delta_b.tool_calls} == {0}
547
+
548
+ def test_dotted_function_name(self, gpt_oss_tokenizer, chat_request):
549
+ parser = HarmonyParser(gpt_oss_tokenizer)
550
+
551
+ delta = parser.parse_delta(
552
+ delta_text="",
553
+ delta_token_ids=encode_output(
554
+ "<|channel|>analysis<|message|>Compute this<|end|>"
555
+ "<|start|>assistant to=math.sum<|channel|>commentary"
556
+ '<|constrain|>json<|message|>{"a": 2, "b": 3}'
557
+ ),
558
+ request=chat_request,
559
+ finished=False,
560
+ )
561
+
562
+ assert delta is not None
563
+ assert [tool.function.name for tool in tool_call_headers(delta)] == ["math.sum"]
564
+ assert {tool.index for tool in delta.tool_calls} == {0}
565
+
566
+ @pytest.mark.parametrize("recipient", ["assistant", "browser"])
567
+ def test_builtin_recipient_skipped(
568
+ self,
569
+ gpt_oss_tokenizer,
570
+ chat_request,
571
+ recipient,
572
+ ):
573
+ parser = HarmonyParser(gpt_oss_tokenizer)
574
+ prompt = [Message.from_role_and_content(Role.USER, "Hello")]
575
+ response = [tool_call(recipient, "Ignore this", content_type=None)]
576
+
577
+ delta = parser.parse_delta(
578
+ delta_text="",
579
+ delta_token_ids=get_model_output_tokens(prompt, response),
580
+ request=chat_request,
581
+ finished=False,
582
+ )
583
+
584
+ assert delta is None
585
+
586
+ def test_cross_channel_with_tool(self, gpt_oss_tokenizer, chat_request):
587
+ parser = HarmonyParser(gpt_oss_tokenizer)
588
+
589
+ delta = parser.parse_delta(
590
+ delta_text="",
591
+ delta_token_ids=encode_output(
592
+ "<|channel|>analysis<|message|>Reasoning about query...<|end|>"
593
+ "<|start|>assistant to=functions.search<|channel|>commentary"
594
+ '<|constrain|>json<|message|>{"query": "vllm"}<|call|>'
595
+ "<|start|>assistant<|channel|>final<|message|>Done"
596
+ ),
597
+ request=chat_request,
598
+ finished=False,
599
+ )
600
+
601
+ assert delta is not None
602
+ assert delta.reasoning == "Reasoning about query..."
603
+ assert delta.content == "Done"
604
+ assert [tool.function.name for tool in tool_call_headers(delta)] == ["search"]
605
+ assert combined_tool_arguments(delta) == {0: '{"query": "vllm"}'}
606
+
607
+ def test_tool_index_across_calls(self, gpt_oss_tokenizer, chat_request):
608
+ parser = HarmonyParser(gpt_oss_tokenizer)
609
+
610
+ first_delta = parser.parse_delta(
611
+ delta_text="",
612
+ delta_token_ids=encode_output(
613
+ "<|channel|>analysis<|message|>Thinking<|end|>"
614
+ "<|start|>assistant to=functions.get_weather<|channel|>commentary"
615
+ '<|constrain|>json<|message|>{"location": "Paris"}<|call|>'
616
+ ),
617
+ request=chat_request,
618
+ finished=False,
619
+ )
620
+ second_delta = parser.parse_delta(
621
+ delta_text="",
622
+ delta_token_ids=encode_output(
623
+ "<|start|>assistant to=functions.get_time<|channel|>commentary"
624
+ '<|constrain|>json<|message|>{"timezone": "UTC"}<|call|>'
625
+ ),
626
+ request=chat_request,
627
+ finished=False,
628
+ )
629
+
630
+ assert [tool.index for tool in tool_call_headers(first_delta)] == [0]
631
+ assert [tool.index for tool in tool_call_headers(second_delta)] == [1]
632
+ assert [tool.function.name for tool in tool_call_headers(second_delta)] == [
633
+ "get_time"
634
+ ]
635
+
636
+ def test_multi_tool_interleaved(self, gpt_oss_tokenizer, chat_request):
637
+ parser = HarmonyParser(gpt_oss_tokenizer)
638
+
639
+ first_delta = parser.parse_delta(
640
+ delta_text="",
641
+ delta_token_ids=encode_output(
642
+ "<|channel|>analysis<|message|>Plan<|end|>"
643
+ "<|start|>assistant to=functions.tool_a<|channel|>commentary"
644
+ '<|constrain|>json<|message|>{"a": 1}<|call|>'
645
+ "<|start|>assistant to=functions.tool_b<|channel|>commentary"
646
+ '<|constrain|>json<|message|>{"b": '
647
+ ),
648
+ request=chat_request,
649
+ finished=False,
650
+ )
651
+ second_delta = parser.parse_delta(
652
+ delta_text="",
653
+ delta_token_ids=encode_output("2"),
654
+ request=chat_request,
655
+ finished=False,
656
+ )
657
+ third_delta = parser.parse_delta(
658
+ delta_text="",
659
+ delta_token_ids=encode_output(
660
+ "}<|call|><|start|>assistant<|channel|>final<|message|>Done<|end|>"
661
+ "<|start|>assistant to=functions.tool_c<|channel|>commentary"
662
+ '<|constrain|>json<|message|>{"c": 3}'
663
+ ),
664
+ request=chat_request,
665
+ finished=False,
666
+ )
667
+
668
+ assert [tool.index for tool in tool_call_headers(first_delta)] == [0, 1]
669
+ assert combined_tool_arguments(first_delta) == {
670
+ 0: '{"a": 1}',
671
+ 1: '{"b": ',
672
+ }
673
+
674
+ assert second_delta is not None
675
+ assert [tool.index for tool in tool_call_payloads(second_delta)] == [1]
676
+ assert combined_tool_arguments(second_delta) == {1: "2"}
677
+
678
+ assert third_delta is not None
679
+ assert third_delta.content == "Done"
680
+ assert combined_tool_arguments(third_delta) == {
681
+ 1: "}",
682
+ 2: '{"c": 3}',
683
+ }
684
+ assert [tool.index for tool in tool_call_headers(third_delta)] == [2]
685
+
686
+
687
class TestProcessChunk:
688
def test_empty(self, harmony_parser):
689
result = harmony_parser.process_chunk([])
@@ -405,7 +695,9 @@ class TestProcessChunk:
695
encode_output("<|channel|>final<|message|>Hello")
696
)
697
408
- assert visible_segments(result) == [("final", None, "Hello")]
698
+ assert [
699
+ (s.channel, s.recipient, s.delta) for s in result.segments if s.delta
700
+ ] == [("final", None, "Hello")]
701
702
def test_cross_channel(self, harmony_parser):
703
result = harmony_parser.process_chunk(
@@ -415,24 +707,13 @@ class TestProcessChunk:
707
)
708
)
709
418
- assert visible_segments(result) == [
710
+ assert [
711
+ (s.channel, s.recipient, s.delta) for s in result.segments if s.delta
712
+ ] == [
713
("analysis", None, "Think"),
714
("final", None, "Answer"),
715
]
716
423
- def test_boundary_detection(self, harmony_parser):
424
- result = harmony_parser.process_chunk(
425
- encode_output("<|channel|>final<|message|>Done<|end|>")
426
- )
427
-
428
- boundary_segments = [
429
- segment for segment in result.segments if segment.is_boundary
430
- ]
431
- assert len(boundary_segments) == 1
432
- assert boundary_segments[0].completed_message is not None
433
- assert boundary_segments[0].completed_message.channel == "final"
434
- assert get_text(boundary_segments[0].completed_message) == "Done"
435
-
717
def test_multi_boundary(self, harmony_parser):
718
result = harmony_parser.process_chunk(
719
encode_output(
@@ -442,11 +723,14 @@ class TestProcessChunk:
723
)
724
725
boundary_segments = [
445
- segment for segment in result.segments if segment.is_boundary
726
+ segment
727
+ for segment in result.segments
728
+ if segment.completed_message is not None
729
]
730
assert [
448
- get_text(segment.completed_message) for segment in boundary_segments
731
+ (segment.completed_message.channel, get_text(segment.completed_message))
732
+ for segment in boundary_segments
733
] == [
450
- "One",
451
- "Two",
734
+ ("analysis", "One"),
735
+ ("final", "Two"),
736
]
vllm/entrypoints/openai/chat_completion/serving.py
+18
-55
@@ -33,10 +33,6 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
33
ChatCompletionStreamResponse,
34
ChatMessage,
35
)
36
-from vllm.entrypoints.openai.chat_completion.stream_harmony import (
37
- TokenState,
38
- extract_harmony_streaming_delta,
39
-)
36
from vllm.entrypoints.openai.engine.protocol import (
37
DeltaMessage,
38
ErrorResponse,
@@ -52,9 +48,6 @@ from vllm.entrypoints.openai.engine.serving import (
48
clamp_prompt_logprobs,
49
)
50
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
55
-from vllm.entrypoints.openai.parser.harmony_utils import (
56
- get_streamable_parser_for_assistant,
57
-)
51
from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage
52
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
53
from vllm.entrypoints.serve.utils.tool_calls_utils import (
@@ -155,7 +148,6 @@ class OpenAIServingChat(OpenAIServing):
148
if mc.generation_config not in ("auto", "vllm")
149
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
150
)
158
- self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
151
self.tool_call_id_type = get_tool_call_id_type(self.model_config)
152
153
# NOTE(woosuk): While OpenAI's chat completion API supports browsing
@@ -408,11 +400,6 @@ class OpenAIServingChat(OpenAIServing):
400
finish_reason_sent = [False] * num_choices
401
num_prompt_tokens = 0
402
num_cached_tokens = None
411
- if self.use_harmony:
412
- harmony_parsers = [
413
- get_streamable_parser_for_assistant() for _ in range(num_choices)
414
- ]
415
- harmony_tools_streamed = [False] * num_choices
403
tools_streamed = [False] * num_choices
404
405
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
@@ -443,6 +430,7 @@ class OpenAIServingChat(OpenAIServing):
430
]
431
for p in parsers:
432
if p is not None:
433
+ # NOTE: HarmonyParser ignores _stream_state (uses its own FSM).
434
p._stream_state.tool_call_id_type = self.tool_call_id_type
435
p._stream_state.history_tool_call_cnt = history_tool_call_cnt
436
else:
@@ -572,32 +560,7 @@ class OpenAIServingChat(OpenAIServing):
560
else:
561
logprobs = None
562
575
- if self.use_harmony:
576
- harmony_parser = harmony_parsers[i]
577
- prev_recipient = harmony_parser.current_recipient
578
-
579
- # Track accumulated content per token with their state
580
- token_states: list[TokenState] = []
581
- for token_id in output.token_ids:
582
- harmony_parser.process(token_id)
583
- token_delta = harmony_parser.last_content_delta or ""
584
- token_states.append(
585
- TokenState(
586
- harmony_parser.current_channel,
587
- harmony_parser.current_recipient,
588
- token_delta,
589
- )
590
- )
591
- delta_text = "".join(delta for _, _, delta in token_states)
592
- cur_channel = harmony_parser.current_channel
593
-
594
- # handle the case where several tokens where generated at once
595
- # including the final token, leading to a delta in the text
596
- # but the current channel to be empty (start state)
597
- if not cur_channel and delta_text:
598
- cur_channel = "final"
599
- else:
600
- delta_text = output.text
563
+ delta_text = output.text
564
565
if (
566
not delta_text
@@ -609,17 +572,7 @@ class OpenAIServingChat(OpenAIServing):
572
573
delta_message: DeltaMessage | None
574
612
- if self.use_harmony:
613
- delta_message, tools_streamed_flag = (
614
- extract_harmony_streaming_delta(
615
- harmony_parser=harmony_parser,
616
- token_states=token_states,
617
- prev_recipient=prev_recipient,
618
- include_reasoning=request.include_reasoning,
619
- )
620
- )
621
- harmony_tools_streamed[i] |= tools_streamed_flag
622
- elif parser is not None:
575
+ if parser is not None:
576
delta_message = parser.parse_delta(
577
delta_text=delta_text,
578
delta_token_ids=as_list(output.token_ids),
@@ -627,8 +580,20 @@ class OpenAIServingChat(OpenAIServing):
580
prompt_token_ids=res.prompt_token_ids,
581
finished=output.finish_reason is not None,
582
)
630
- if delta_message and delta_message.tool_calls:
631
- tools_streamed[i] = True
583
+ if delta_message is not None:
584
+ if delta_message.tool_calls:
585
+ tools_streamed[i] = True
586
+
587
+ if (
588
+ delta_message.reasoning
589
+ and not request.include_reasoning
590
+ ):
591
+ delta_message.reasoning = None
592
+ if not (
593
+ delta_message.content or delta_message.tool_calls
594
+ ):
595
+ delta_message = None
596
+
597
# handle streaming just a content delta (no parsers)
598
else:
599
delta_message = DeltaMessage(content=delta_text)
@@ -706,9 +671,7 @@ class OpenAIServingChat(OpenAIServing):
671
# finish_reason is:
672
# "tool_calls" for "auto" or "required" tool calls,
673
# and "stop" for named tool calls.
709
- if (tools_streamed[i] and not tool_choice_function_name) or (
710
- self.use_harmony and harmony_tools_streamed[i]
711
- ):
674
+ if tools_streamed[i] and not tool_choice_function_name:
675
finish_reason_ = "tool_calls"
676
else:
677
finish_reason_ = (
vllm/entrypoints/openai/chat_completion/stream_harmony.py
deleted
-167
@@ -1,167 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""
4
-Harmony-specific streaming delta extraction for chat completions.
5
-
6
-This module handles the extraction of DeltaMessage objects from
7
-harmony parser state during streaming chat completions.
8
-"""
9
-
10
-from typing import NamedTuple
11
-
12
-from openai_harmony import StreamableParser
13
-
14
-from vllm.entrypoints.chat_utils import make_tool_call_id
15
-from vllm.entrypoints.openai.engine.protocol import (
16
- DeltaFunctionCall,
17
- DeltaMessage,
18
- DeltaToolCall,
19
-)
20
-from vllm.entrypoints.openai.parser.harmony_utils import (
21
- extract_function_from_recipient,
22
- is_function_recipient,
23
-)
24
-
25
-
26
-class TokenState(NamedTuple):
27
- channel: str | None
28
- recipient: str | None
29
- text: str
30
-
31
-
32
-def extract_harmony_streaming_delta(
33
- harmony_parser: StreamableParser,
34
- token_states: list[TokenState],
35
- prev_recipient: str | None,
36
- include_reasoning: bool,
37
-) -> tuple[DeltaMessage | None, bool]:
38
- """
39
- Extract a DeltaMessage from harmony parser state during streaming.
40
-
41
- Args:
42
- harmony_parser: The StreamableParser instance tracking parse state
43
- token_states: List of TokenState tuples for each token
44
- prev_recipient: Previous recipient for detecting tool call transitions
45
- include_reasoning: Whether to include reasoning content
46
-
47
- Returns:
48
- A tuple of (DeltaMessage or None, tools_streamed_flag)
49
- """
50
-
51
- if not token_states:
52
- return None, False
53
-
54
- tools_streamed = False
55
-
56
- # Group consecutive tokens with same channel/recipient
57
- groups: list[TokenState] = []
58
-
59
- current_channel = token_states[0].channel
60
- current_recipient = token_states[0].recipient
61
- current_text = token_states[0].text
62
-
63
- for i in range(1, len(token_states)):
64
- state = token_states[i]
65
- if state.channel == current_channel and state.recipient == current_recipient:
66
- current_text += state.text
67
- else:
68
- groups.append(TokenState(current_channel, current_recipient, current_text))
69
- current_channel = state.channel
70
- current_recipient = state.recipient
71
- current_text = state.text
72
-
73
- groups.append(TokenState(current_channel, current_recipient, current_text))
74
-
75
- # Process each group and create delta messages
76
- delta_message = None
77
- combined_content = ""
78
- combined_reasoning = ""
79
- tool_messages = []
80
- content_encountered = False
81
-
82
- # Calculate base_index once before the loop
83
- # This counts completed tool calls in messages
84
- base_index = 0
85
- for msg in harmony_parser.messages:
86
- if msg.recipient and is_function_recipient(msg.recipient):
87
- base_index += 1
88
-
89
- # If there's an ongoing tool call from previous chunk,
90
- # the next new tool call starts at base_index + 1
91
- if prev_recipient and is_function_recipient(prev_recipient):
92
- next_tool_index = base_index + 1
93
- # Ongoing call is at base_index
94
- ongoing_tool_index = base_index
95
- else:
96
- # No ongoing call, next new call is at base_index
97
- next_tool_index = base_index
98
- ongoing_tool_index = None
99
-
100
- for group in groups:
101
- if group.channel == "final":
102
- combined_content += group.text
103
- content_encountered = True
104
- elif group.recipient and is_function_recipient(group.recipient):
105
- opened_new_call = False
106
- if prev_recipient != group.recipient:
107
- # New tool call - emit the opening message
108
- tool_name = extract_function_from_recipient(group.recipient)
109
- tool_messages.append(
110
- DeltaToolCall(
111
- id=make_tool_call_id(),
112
- type="function",
113
- function=DeltaFunctionCall(
114
- name=tool_name,
115
- arguments="",
116
- ),
117
- index=next_tool_index,
118
- )
119
- )
120
- opened_new_call = True
121
- prev_recipient = group.recipient
122
- # Increment for subsequent new tool calls
123
- next_tool_index += 1
124
-
125
- if group.text:
126
- # Stream arguments for the ongoing tool call
127
- if opened_new_call:
128
- # Just opened in this group
129
- tool_call_index = next_tool_index - 1
130
- else:
131
- # Continuing from previous chunk
132
- # If ongoing_tool_index is None here, it means
133
- # we're continuing a call but prev_recipient
134
- # wasn't a function. Use base_index.
135
- tool_call_index = (
136
- ongoing_tool_index
137
- if ongoing_tool_index is not None
138
- else base_index
139
- )
140
- tool_messages.append(
141
- DeltaToolCall(
142
- index=tool_call_index,
143
- function=DeltaFunctionCall(arguments=group.text),
144
- )
145
- )
146
- elif group.channel == "commentary" and group.recipient is None:
147
- # Tool call preambles meant to be shown to the user
148
- combined_content += group.text
149
- content_encountered = True
150
- elif group.channel == "analysis" and include_reasoning:
151
- combined_reasoning += group.text
152
-
153
- # Combine all non-empty fields into a single message
154
- if content_encountered or combined_reasoning or tool_messages:
155
- delta_kwargs: dict[str, str | list[DeltaToolCall]] = {}
156
- if content_encountered:
157
- delta_kwargs["content"] = combined_content
158
- if combined_reasoning:
159
- delta_kwargs["reasoning"] = combined_reasoning
160
- if tool_messages:
161
- delta_kwargs["tool_calls"] = tool_messages
162
- tools_streamed = True
163
- delta_message = DeltaMessage(**delta_kwargs)
164
- else:
165
- delta_message = None
166
-
167
- return delta_message, tools_streamed
vllm/parser/harmony.py
+67
-15
@@ -9,9 +9,12 @@ from dataclasses import dataclass
9
from enum import Enum, auto
10
from typing import TYPE_CHECKING, NamedTuple
11
12
+from vllm.entrypoints.chat_utils import make_tool_call_id
13
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
14
from vllm.entrypoints.openai.engine.protocol import (
15
+ DeltaFunctionCall,
16
DeltaMessage,
17
+ DeltaToolCall,
18
FunctionCall,
19
)
20
from vllm.entrypoints.openai.parser.harmony_utils import (
@@ -52,7 +55,6 @@ class Segment(NamedTuple):
55
channel: str | None
56
recipient: str | None
57
delta: str
55
- is_boundary: bool = False
58
completed_message: Message | None = None
59
60
@@ -81,10 +83,8 @@ class HarmonyParser(DelegatingParser):
83
)
84
85
self._harmony_parser = get_streamable_parser_for_assistant()
84
-
85
- @property
86
- def messages(self) -> list[Message]:
87
- return self._harmony_parser.messages
86
+ self._next_tool_call_index = 0
87
+ self._num_processed_messages = 0
88
89
@property
90
def state(self) -> HarmonyStreamState:
@@ -194,17 +194,69 @@ class HarmonyParser(DelegatingParser):
194
*,
195
finished: bool,
196
) -> DeltaMessage | None:
197
- raise NotImplementedError(
198
- "HarmonyParser streaming parsing is deferred. "
199
- "Use the existing harmony streaming path."
200
- )
197
+ prev_recipient = self.current_recipient
198
+ result = self.process_chunk(delta_token_ids)
199
+ combined_content = ""
200
+ combined_reasoning = ""
201
+ tool_messages: list[DeltaToolCall] = []
202
+
203
+ for segment in result.segments:
204
+ if segment.completed_message is not None:
205
+ prev_recipient = None
206
+ continue
207
+
208
+ segment_type = _SegmentType.from_channel_and_recipient(
209
+ segment.channel, segment.recipient
210
+ )
211
+ match segment_type:
212
+ case _SegmentType.REASONING:
213
+ combined_reasoning += segment.delta
214
+ case _SegmentType.CONTENT:
215
+ combined_content += segment.delta
216
+ case _SegmentType.TOOL:
217
+ assert segment.recipient is not None
218
+ if prev_recipient != segment.recipient:
219
+ tool_name = extract_function_from_recipient(segment.recipient)
220
+ tool_messages.append(
221
+ DeltaToolCall(
222
+ # HarmonyParser does not use _stream_state;
223
+ # "random" tool_call_id_type is always used
224
+ id=make_tool_call_id(),
225
+ type="function",
226
+ function=DeltaFunctionCall(
227
+ name=tool_name,
228
+ arguments=segment.delta,
229
+ ),
230
+ index=self._next_tool_call_index,
231
+ )
232
+ )
233
+ self._next_tool_call_index += 1
234
+ prev_recipient = segment.recipient
235
+ elif segment.delta:
236
+ tool_call_index = self._next_tool_call_index - 1
237
+ tool_messages.append(
238
+ DeltaToolCall(
239
+ index=tool_call_index,
240
+ function=DeltaFunctionCall(arguments=segment.delta),
241
+ )
242
+ )
243
+
244
+ if not combined_content and not combined_reasoning and not tool_messages:
245
+ return None
246
+
247
+ delta_message = DeltaMessage()
248
+ if combined_content:
249
+ delta_message.content = combined_content
250
+ if combined_reasoning:
251
+ delta_message.reasoning = combined_reasoning
252
+ if tool_messages:
253
+ delta_message.tool_calls = tool_messages
254
+ return delta_message
255
256
def process_chunk(self, token_ids: Sequence[int]) -> ChunkResult:
257
if not token_ids:
258
return ChunkResult(segments=[], reasoning_token_count=0)
259
206
- from openai_harmony import StreamState
207
-
260
segments: list[Segment] = []
261
reasoning_token_count = 0
262
for token_id in token_ids:
@@ -213,9 +265,10 @@ class HarmonyParser(DelegatingParser):
265
recipient = self.current_recipient
266
delta = self._harmony_parser.last_content_delta or ""
267
completed_message = None
216
- is_boundary = self.state == StreamState.EXPECT_START
217
- if is_boundary and self.messages:
218
- completed_message = self.messages[-1]
268
+ _messages = self._harmony_parser.messages
269
+ if len(_messages) > self._num_processed_messages:
270
+ completed_message = _messages[self._num_processed_messages]
271
+ self._num_processed_messages += 1
272
273
if channel == "analysis" or (
274
channel == "commentary" and recipient is not None
@@ -227,7 +280,6 @@ class HarmonyParser(DelegatingParser):
280
channel=channel,
281
recipient=recipient,
282
delta=delta,
230
- is_boundary=is_boundary,
283
completed_message=completed_message,
284
)
285
)