A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 05:00 UTC
[Frontend][Bug] allow tool calls in analysis channel (#28139)
Signed-off-by: Marko Rosenmueller <5467316+dr75@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
Marko Rosenmueller committed
Dec 19, 2025 at 11:47 UTC
455949675d1808e4fafd54c17a94aec0c3e30fff
3 files changed
+327
-58
tests/entrypoints/openai/test_serving_chat_stream_harmony.py
new
+212
@@ -0,0 +1,212 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+Unit tests for harmony streaming delta extraction.
5
+"""
6
+
7
+from dataclasses import dataclass, field
8
+from unittest.mock import patch
9
+
10
+import pytest
11
+
12
+from vllm.entrypoints.openai.serving_chat_stream_harmony import (
13
+ extract_harmony_streaming_delta,
14
+)
15
+
16
+
17
+@dataclass
18
+class MockMessage:
19
+ """Mock message object for testing."""
20
+
21
+ channel: str | None = None
22
+ recipient: str | None = None
23
+
24
+
25
+@dataclass
26
+class MockStreamableParser:
27
+ """Mock StreamableParser for testing without openai_harmony dependency."""
28
+
29
+ messages: list[MockMessage] = field(default_factory=list)
30
+
31
+
32
+class TestExtractHarmonyStreamingDelta:
33
+ """Tests for extract_harmony_streaming_delta function."""
34
+
35
+ @pytest.mark.parametrize(
36
+ "delta_text,expected_content",
37
+ [
38
+ ("Hello, world!", "Hello, world!"),
39
+ ("", ""),
40
+ ],
41
+ )
42
+ def test_final_channel_returns_content_delta(self, delta_text, expected_content):
43
+ """Test that final channel returns a DeltaMessage with content."""
44
+ parser = MockStreamableParser()
45
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
46
+ harmony_parser=parser,
47
+ cur_channel="final",
48
+ cur_recipient=None,
49
+ prev_recipient=None,
50
+ delta_text=delta_text,
51
+ include_reasoning=False,
52
+ )
53
+
54
+ assert delta_message is not None
55
+ assert delta_message.content == expected_content
56
+ assert tools_streamed is False
57
+
58
+ @pytest.mark.parametrize(
59
+ "include_reasoning,expected_has_message",
60
+ [
61
+ (True, True),
62
+ (False, False),
63
+ ],
64
+ )
65
+ def test_analysis_channel_reasoning(self, include_reasoning, expected_has_message):
66
+ """Test analysis channel respects include_reasoning flag."""
67
+ parser = MockStreamableParser()
68
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
69
+ harmony_parser=parser,
70
+ cur_channel="analysis",
71
+ cur_recipient=None,
72
+ prev_recipient=None,
73
+ delta_text="Let me think...",
74
+ include_reasoning=include_reasoning,
75
+ )
76
+
77
+ if expected_has_message:
78
+ assert delta_message is not None
79
+ assert delta_message.reasoning == "Let me think..."
80
+ else:
81
+ assert delta_message is None
82
+ assert tools_streamed is False
83
+
84
+ @pytest.mark.parametrize("channel", ["commentary", "analysis"])
85
+ @patch("vllm.entrypoints.openai.serving_chat_stream_harmony.make_tool_call_id")
86
+ def test_new_tool_call(self, mock_make_tool_call_id, channel):
87
+ """Test new tool call creation when recipient changes."""
88
+ mock_make_tool_call_id.return_value = "call_test123"
89
+ parser = MockStreamableParser()
90
+
91
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
92
+ harmony_parser=parser,
93
+ cur_channel=channel,
94
+ cur_recipient="functions.get_weather",
95
+ prev_recipient=None,
96
+ delta_text="",
97
+ include_reasoning=False,
98
+ )
99
+
100
+ assert delta_message is not None
101
+ assert len(delta_message.tool_calls) == 1
102
+ tool_call = delta_message.tool_calls[0]
103
+ assert tool_call.id == "call_test123"
104
+ assert tool_call.type == "function"
105
+ assert tool_call.function.name == "get_weather"
106
+ assert tool_call.function.arguments == ""
107
+ assert tool_call.index == 0
108
+ assert tools_streamed is True
109
+
110
+ @pytest.mark.parametrize("channel", ["commentary", "analysis"])
111
+ def test_tool_call_argument_streaming(self, channel):
112
+ """Test streaming tool call arguments (same recipient)."""
113
+ parser = MockStreamableParser()
114
+
115
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
116
+ harmony_parser=parser,
117
+ cur_channel=channel,
118
+ cur_recipient="functions.get_weather",
119
+ prev_recipient="functions.get_weather",
120
+ delta_text='{"location": "Paris"}',
121
+ include_reasoning=False,
122
+ )
123
+
124
+ assert delta_message is not None
125
+ tool_call = delta_message.tool_calls[0]
126
+ assert tool_call.id is None
127
+ assert tool_call.function.arguments == '{"location": "Paris"}'
128
+ assert tool_call.index == 0
129
+ assert tools_streamed is True
130
+
131
+ @pytest.mark.parametrize("channel", ["commentary", "analysis"])
132
+ def test_tool_call_empty_arguments_returns_none(self, channel):
133
+ """Test empty delta_text with same recipient returns None."""
134
+ parser = MockStreamableParser()
135
+
136
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
137
+ harmony_parser=parser,
138
+ cur_channel=channel,
139
+ cur_recipient="functions.get_weather",
140
+ prev_recipient="functions.get_weather",
141
+ delta_text="",
142
+ include_reasoning=False,
143
+ )
144
+
145
+ assert delta_message is None
146
+ assert tools_streamed is False
147
+
148
+ def test_tool_call_index_from_previous_messages(self):
149
+ """Test tool call index accounts for previous function messages."""
150
+ messages = [
151
+ MockMessage(channel="analysis", recipient=None), # Not counted
152
+ MockMessage(channel="commentary", recipient="functions.tool1"), # Counted
153
+ MockMessage(channel="final", recipient=None), # Not counted
154
+ ]
155
+ parser = MockStreamableParser(messages=messages)
156
+
157
+ delta_message, _ = extract_harmony_streaming_delta(
158
+ harmony_parser=parser,
159
+ cur_channel="commentary",
160
+ cur_recipient="functions.tool2",
161
+ prev_recipient="functions.tool2",
162
+ delta_text="args",
163
+ include_reasoning=False,
164
+ )
165
+
166
+ assert delta_message.tool_calls[0].index == 1
167
+
168
+ @pytest.mark.parametrize(
169
+ "channel,recipient",
170
+ [
171
+ ("commentary", None),
172
+ ("commentary", "browser.search"),
173
+ ],
174
+ )
175
+ def test_returns_tool_call_preambles(self, channel, recipient):
176
+ """Test that invalid channel/recipient combinations return None."""
177
+ parser = MockStreamableParser()
178
+ delta_text = "some text"
179
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
180
+ harmony_parser=parser,
181
+ cur_channel=channel,
182
+ cur_recipient=recipient,
183
+ prev_recipient=None,
184
+ delta_text=delta_text,
185
+ include_reasoning=True,
186
+ )
187
+
188
+ assert delta_message.content == delta_text
189
+ assert tools_streamed is False
190
+
191
+ @pytest.mark.parametrize(
192
+ "channel,recipient",
193
+ [
194
+ (None, None),
195
+ ("unknown_channel", None),
196
+ ],
197
+ )
198
+ def test_returns_none_for_invalid_inputs(self, channel, recipient):
199
+ """Test that invalid channel/recipient combinations return None."""
200
+ parser = MockStreamableParser()
201
+
202
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
203
+ harmony_parser=parser,
204
+ cur_channel=channel,
205
+ cur_recipient=recipient,
206
+ prev_recipient=None,
207
+ delta_text="some text",
208
+ include_reasoning=True,
209
+ )
210
+
211
+ assert delta_message is None
212
+ assert tools_streamed is False
vllm/entrypoints/openai/serving_chat.py
+14
-58
@@ -51,6 +51,9 @@ from vllm.entrypoints.openai.protocol import (
51
ToolCall,
52
UsageInfo,
53
)
54
+from vllm.entrypoints.openai.serving_chat_stream_harmony import (
55
+ extract_harmony_streaming_delta,
56
+)
57
from vllm.entrypoints.openai.serving_engine import (
58
GenerationError,
59
OpenAIServing,
@@ -837,64 +840,17 @@ class OpenAIServingChat(OpenAIServing):
840
current_token_ids = as_list(output.token_ids)
841
842
if self.use_harmony:
840
- if cur_channel == "final":
841
- delta_message = DeltaMessage(content=delta_text)
842
- elif cur_channel == "analysis":
843
- if request.include_reasoning:
844
- delta_message = DeltaMessage(reasoning=delta_text)
845
- else:
846
- delta_message = None
847
- elif (
848
- cur_channel == "commentary"
849
- and cur_recipient
850
- and cur_recipient.startswith("functions.")
851
- ):
852
- # Count completed tool calls to determine index
853
- base_index = 0
854
- for msg in harmony_parser.messages:
855
- if (
856
- msg.channel == "commentary"
857
- and msg.recipient
858
- and msg.recipient.startswith("functions.")
859
- ):
860
- base_index += 1
861
-
862
- if prev_recipient != cur_recipient:
863
- tool_name = cur_recipient.split("functions.", 1)[1]
864
- delta_message = DeltaMessage(
865
- tool_calls=[
866
- DeltaToolCall(
867
- id=make_tool_call_id(),
868
- type="function",
869
- function=DeltaFunctionCall(
870
- name=tool_name,
871
- arguments="",
872
- ),
873
- index=base_index,
874
- )
875
- ]
876
- )
877
- elif delta_text:
878
- delta_message = DeltaMessage(
879
- tool_calls=[
880
- DeltaToolCall(
881
- index=base_index,
882
- function=DeltaFunctionCall(
883
- arguments=delta_text
884
- ),
885
- )
886
- ]
887
- )
888
- else:
889
- delta_message = None
890
-
891
- if delta_message is not None:
892
- harmony_tools_streamed[i] = True
893
- elif cur_channel == "commentary":
894
- # Tool call preambles meant to be shown to the user
895
- delta_message = DeltaMessage(content=delta_text)
896
- else:
897
- delta_message = None
843
+ delta_message, tools_streamed_flag = (
844
+ extract_harmony_streaming_delta(
845
+ harmony_parser=harmony_parser,
846
+ cur_channel=cur_channel,
847
+ cur_recipient=cur_recipient,
848
+ prev_recipient=prev_recipient,
849
+ delta_text=delta_text,
850
+ include_reasoning=request.include_reasoning,
851
+ )
852
+ )
853
+ harmony_tools_streamed[i] |= tools_streamed_flag
854
# handle streaming deltas for tools with named tool_choice
855
elif tool_choice_function_name:
856
if (
vllm/entrypoints/openai/serving_chat_stream_harmony.py
new
+101
@@ -0,0 +1,101 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+Harmony-specific streaming delta extraction for chat completions.
5
+
6
+This module handles the extraction of DeltaMessage objects from
7
+harmony parser state during streaming chat completions.
8
+"""
9
+
10
+from openai_harmony import StreamableParser
11
+
12
+from vllm.entrypoints.chat_utils import make_tool_call_id
13
+from vllm.entrypoints.openai.protocol import (
14
+ DeltaFunctionCall,
15
+ DeltaMessage,
16
+ DeltaToolCall,
17
+)
18
+
19
+
20
+def extract_harmony_streaming_delta(
21
+ harmony_parser: StreamableParser,
22
+ cur_channel: str | None,
23
+ cur_recipient: str | None,
24
+ prev_recipient: str | None,
25
+ delta_text: str,
26
+ include_reasoning: bool,
27
+) -> tuple[DeltaMessage | None, bool]:
28
+ """
29
+ Extract a DeltaMessage from harmony parser state during streaming.
30
+
31
+ Args:
32
+ harmony_parser: The StreamableParser instance tracking parse state
33
+ cur_channel: Current channel ("final", "analysis", "commentary", etc.)
34
+ cur_recipient: Current recipient (e.g., "functions.my_func")
35
+ prev_recipient: Previous recipient for detecting tool call transitions
36
+ delta_text: The text delta to include in the message
37
+ include_reasoning: Whether to include reasoning content
38
+
39
+ Returns:
40
+ A tuple of (DeltaMessage or None, tools_streamed_flag)
41
+ """
42
+ tools_streamed = False
43
+
44
+ if cur_channel == "final":
45
+ delta_message = DeltaMessage(content=delta_text)
46
+ elif (
47
+ (cur_channel == "commentary" or cur_channel == "analysis")
48
+ and cur_recipient
49
+ and cur_recipient.startswith("functions.")
50
+ ):
51
+ # Count completed tool calls to determine index
52
+ base_index = 0
53
+ for msg in harmony_parser.messages:
54
+ if (
55
+ (msg.channel == "commentary" or msg.channel == "analysis")
56
+ and msg.recipient
57
+ and msg.recipient.startswith("functions.")
58
+ ):
59
+ base_index += 1
60
+
61
+ if prev_recipient != cur_recipient:
62
+ tool_name = cur_recipient.split("functions.", 1)[1]
63
+ delta_message = DeltaMessage(
64
+ tool_calls=[
65
+ DeltaToolCall(
66
+ id=make_tool_call_id(),
67
+ type="function",
68
+ function=DeltaFunctionCall(
69
+ name=tool_name,
70
+ arguments="",
71
+ ),
72
+ index=base_index,
73
+ )
74
+ ]
75
+ )
76
+ elif delta_text:
77
+ delta_message = DeltaMessage(
78
+ tool_calls=[
79
+ DeltaToolCall(
80
+ index=base_index,
81
+ function=DeltaFunctionCall(arguments=delta_text),
82
+ )
83
+ ]
84
+ )
85
+ else:
86
+ delta_message = None
87
+
88
+ if delta_message is not None:
89
+ tools_streamed = True
90
+ elif cur_channel == "commentary":
91
+ # Tool call preambles meant to be shown to the user
92
+ delta_message = DeltaMessage(content=delta_text)
93
+ elif cur_channel == "analysis":
94
+ if include_reasoning:
95
+ delta_message = DeltaMessage(reasoning=delta_text)
96
+ else:
97
+ delta_message = None
98
+ else:
99
+ delta_message = None
100
+
101
+ return delta_message, tools_streamed