A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 17:45 UTC
[Bugfix] Handle real-world gpt-oss tool call output in Harmony parsing (#42454)
Signed-off-by: Ben Browning <bbrownin@redhat.com>
Ben Browning committed
May 13, 2026 at 13:54 UTC
0f69128a3773a2c622d958631c9fde8d68a467d8
12 files changed
+801
-50
tests/entrypoints/openai/chat_completion/test_serving_chat_stream_harmony.py
+184
@@ -199,12 +199,107 @@ class TestExtractHarmonyStreamingDelta:
199
assert delta_message.content == delta_text
200
assert tools_streamed is False
201
202
+ @pytest.mark.parametrize("channel", ["commentary", "analysis"])
203
+ @patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
204
+ def test_new_tool_call_without_functions_prefix(
205
+ self, mock_make_tool_call_id, channel
206
+ ):
207
+ mock_make_tool_call_id.return_value = "call_bare123"
208
+ parser = MockStreamableParser()
209
+
210
+ token_states = [TokenState(channel=channel, recipient="get_weather", text="")]
211
+
212
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
213
+ harmony_parser=parser,
214
+ token_states=token_states,
215
+ prev_recipient=None,
216
+ include_reasoning=False,
217
+ )
218
+
219
+ assert delta_message is not None
220
+ assert len(delta_message.tool_calls) == 1
221
+ tool_call = delta_message.tool_calls[0]
222
+ assert tool_call.id == "call_bare123"
223
+ assert tool_call.type == "function"
224
+ assert tool_call.function.name == "get_weather"
225
+ assert tool_call.function.arguments == ""
226
+ assert tool_call.index == 0
227
+ assert tools_streamed is True
228
+
229
+ @pytest.mark.parametrize("channel", ["commentary", "analysis"])
230
+ def test_tool_call_argument_streaming_without_functions_prefix(self, channel):
231
+ parser = MockStreamableParser()
232
+ args_text = '{"location": "Paris"}'
233
+
234
+ token_states = [
235
+ TokenState(channel=channel, recipient="get_weather", text=args_text)
236
+ ]
237
+
238
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
239
+ harmony_parser=parser,
240
+ token_states=token_states,
241
+ prev_recipient="get_weather",
242
+ include_reasoning=False,
243
+ )
244
+
245
+ assert delta_message is not None
246
+ tool_call = delta_message.tool_calls[0]
247
+ assert tool_call.id is None
248
+ assert tool_call.function.arguments == args_text
249
+ assert tool_call.index == 0
250
+ assert tools_streamed is True
251
+
252
+ def test_tool_call_index_from_previous_messages_without_functions_prefix(self):
253
+ messages = [
254
+ MockMessage(channel="commentary", recipient="tool1"),
255
+ ]
256
+ parser = MockStreamableParser(messages=messages)
257
+
258
+ token_states = [
259
+ TokenState(channel="commentary", recipient="tool2", text="args")
260
+ ]
261
+
262
+ delta_message, _ = extract_harmony_streaming_delta(
263
+ harmony_parser=parser,
264
+ token_states=token_states,
265
+ prev_recipient="tool2",
266
+ include_reasoning=False,
267
+ )
268
+
269
+ assert delta_message.tool_calls[0].index == 1
270
+
271
+ @pytest.mark.parametrize("channel", ["commentary", "analysis"])
272
+ @patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
273
+ def test_new_tool_call_dotted_function_name(self, mock_make_tool_call_id, channel):
274
+ mock_make_tool_call_id.return_value = "call_dotted123"
275
+ parser = MockStreamableParser()
276
+
277
+ token_states = [TokenState(channel=channel, recipient="math.sum", text="")]
278
+
279
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
280
+ harmony_parser=parser,
281
+ token_states=token_states,
282
+ prev_recipient=None,
283
+ include_reasoning=False,
284
+ )
285
+
286
+ assert delta_message is not None
287
+ assert len(delta_message.tool_calls) == 1
288
+ tool_call = delta_message.tool_calls[0]
289
+ assert tool_call.id == "call_dotted123"
290
+ assert tool_call.type == "function"
291
+ assert tool_call.function.name == "math.sum"
292
+ assert tool_call.function.arguments == ""
293
+ assert tool_call.index == 0
294
+ assert tools_streamed is True
295
+
296
@pytest.mark.parametrize(
297
"channel,recipient",
298
[
299
(None, None),
300
("unknown_channel", None),
301
("commentary", "browser.search"),
302
+ ("commentary", "assistant"),
303
],
304
)
305
def test_returns_none_for_invalid_inputs(self, channel, recipient):
@@ -348,3 +443,92 @@ class TestExtractHarmonyStreamingDelta:
443
assert tool_c_args.function.arguments == '{"key_c": "val_c"}'
444
445
assert delta_message.content == "Thinking... Thinking again..."
446
+
447
+
448
+class TestToolCallsOnNonStandardChannels:
449
+ """Tool calls are detected by recipient, not channel.
450
+
451
+ Models sometimes emit tool calls on unexpected channels (e.g. ``comment``
452
+ instead of ``commentary``). These tests verify that the streaming delta
453
+ extraction is channel-agnostic for tool call detection.
454
+ """
455
+
456
+ @patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
457
+ def test_prefixed_tool_call_on_comment_channel(self, mock_make_tool_call_id):
458
+ mock_make_tool_call_id.return_value = "call_comment_chan"
459
+ parser = MockStreamableParser()
460
+
461
+ token_states = [
462
+ TokenState(channel="comment", recipient="functions.get_weather", text="")
463
+ ]
464
+
465
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
466
+ harmony_parser=parser,
467
+ token_states=token_states,
468
+ prev_recipient=None,
469
+ include_reasoning=False,
470
+ )
471
+
472
+ assert delta_message is not None
473
+ assert len(delta_message.tool_calls) == 1
474
+ assert delta_message.tool_calls[0].function.name == "get_weather"
475
+ assert tools_streamed is True
476
+
477
+ @patch("vllm.entrypoints.openai.chat_completion.stream_harmony.make_tool_call_id")
478
+ def test_bare_tool_call_on_comment_channel(self, mock_make_tool_call_id):
479
+ mock_make_tool_call_id.return_value = "call_bare_comment"
480
+ parser = MockStreamableParser()
481
+
482
+ token_states = [TokenState(channel="comment", recipient="get_weather", text="")]
483
+
484
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
485
+ harmony_parser=parser,
486
+ token_states=token_states,
487
+ prev_recipient=None,
488
+ include_reasoning=False,
489
+ )
490
+
491
+ assert delta_message is not None
492
+ assert len(delta_message.tool_calls) == 1
493
+ assert delta_message.tool_calls[0].function.name == "get_weather"
494
+ assert tools_streamed is True
495
+
496
+ def test_tool_call_arguments_on_comment_channel(self):
497
+ parser = MockStreamableParser()
498
+ args_text = '{"location": "Paris"}'
499
+
500
+ token_states = [
501
+ TokenState(
502
+ channel="comment", recipient="functions.get_weather", text=args_text
503
+ )
504
+ ]
505
+
506
+ delta_message, tools_streamed = extract_harmony_streaming_delta(
507
+ harmony_parser=parser,
508
+ token_states=token_states,
509
+ prev_recipient="functions.get_weather",
510
+ include_reasoning=False,
511
+ )
512
+
513
+ assert delta_message is not None
514
+ assert delta_message.tool_calls[0].function.arguments == args_text
515
+ assert tools_streamed is True
516
+
517
+ def test_base_index_counts_tool_calls_on_comment_channel(self):
518
+ messages = [
519
+ MockMessage(channel="comment", recipient="functions.tool1"),
520
+ ]
521
+ parser = MockStreamableParser(messages=messages)
522
+
523
+ token_states = [
524
+ TokenState(channel="commentary", recipient="functions.tool2", text="args")
525
+ ]
526
+
527
+ delta_message, _ = extract_harmony_streaming_delta(
528
+ harmony_parser=parser,
529
+ token_states=token_states,
530
+ prev_recipient="functions.tool2",
531
+ include_reasoning=False,
532
+ )
533
+
534
+ assert delta_message.tool_calls[0].index == 1
tests/entrypoints/openai/parser/test_harmony_utils.py
+178
@@ -7,9 +7,11 @@ from openai_harmony import Message, Role
7
from tests.entrypoints.openai.utils import verify_harmony_messages
8
from vllm.entrypoints.openai.parser.harmony_utils import (
9
auto_drop_analysis_messages,
10
+ extract_function_from_recipient,
11
get_encoding,
12
get_system_message,
13
has_custom_tools,
14
+ is_function_recipient,
15
parse_chat_input_to_harmony_message,
16
parse_chat_output,
17
)
@@ -19,6 +21,182 @@ from vllm.entrypoints.openai.responses.harmony import (
21
)
22
23
24
+class TestIsFunctionRecipient:
25
+ @pytest.mark.parametrize(
26
+ "recipient",
27
+ [
28
+ "functions.get_weather",
29
+ "functions.search_web",
30
+ "functions.math.sum",
31
+ ],
32
+ )
33
+ def test_functions_prefix_accepted(self, recipient):
34
+ assert is_function_recipient(recipient) is True
35
+
36
+ @pytest.mark.parametrize(
37
+ "recipient",
38
+ [
39
+ "get_weather",
40
+ "search_web",
41
+ "calculator",
42
+ "my-tool",
43
+ ],
44
+ )
45
+ def test_bare_function_name_accepted(self, recipient):
46
+ assert is_function_recipient(recipient) is True
47
+
48
+ @pytest.mark.parametrize(
49
+ "recipient",
50
+ [
51
+ "assistant",
52
+ ],
53
+ )
54
+ def test_assistant_rejected(self, recipient):
55
+ assert is_function_recipient(recipient) is False
56
+
57
+ @pytest.mark.parametrize(
58
+ "recipient",
59
+ [
60
+ "math.sum",
61
+ "code.run",
62
+ "namespace.tool_name",
63
+ "my.deeply.nested.tool",
64
+ ],
65
+ )
66
+ def test_dotted_function_names_accepted(self, recipient):
67
+ assert is_function_recipient(recipient) is True
68
+
69
+ @pytest.mark.parametrize(
70
+ "recipient",
71
+ [
72
+ "python",
73
+ "browser",
74
+ "container",
75
+ ],
76
+ )
77
+ def test_builtin_tool_names_rejected(self, recipient):
78
+ assert is_function_recipient(recipient) is False
79
+
80
+ @pytest.mark.parametrize(
81
+ "recipient",
82
+ [
83
+ "python.run",
84
+ "python.execute",
85
+ "browser.search",
86
+ "browser.open",
87
+ "container.exec",
88
+ ],
89
+ )
90
+ def test_builtin_dotted_variants_rejected(self, recipient):
91
+ assert is_function_recipient(recipient) is False
92
+
93
+ @pytest.mark.parametrize(
94
+ "recipient",
95
+ [
96
+ "",
97
+ "functions.",
98
+ ],
99
+ )
100
+ def test_empty_recipients_rejected(self, recipient):
101
+ assert is_function_recipient(recipient) is False
102
+
103
+ @pytest.mark.parametrize(
104
+ "recipient",
105
+ [
106
+ "<|start|>",
107
+ "<|end|>",
108
+ "<|channel|>",
109
+ ],
110
+ )
111
+ def test_harmony_tokens_rejected(self, recipient):
112
+ assert is_function_recipient(recipient) is False
113
+
114
+
115
+class TestIsFunctionRecipientWithAllowedNames:
116
+ """Tests for is_function_recipient with allowed_function_tool_names."""
117
+
118
+ def test_prefixed_always_accepted(self):
119
+ """functions. prefix is always accepted regardless of allowed names."""
120
+ fn_names = frozenset({"other_tool"})
121
+ assert is_function_recipient("functions.get_weather", fn_names) is True
122
+
123
+ def test_bare_name_accepted_when_in_allowed_names(self):
124
+ fn_names = frozenset({"get_weather", "search_web"})
125
+ assert is_function_recipient("get_weather", fn_names) is True
126
+ assert is_function_recipient("search_web", fn_names) is True
127
+
128
+ def test_bare_name_rejected_when_not_in_allowed_names(self):
129
+ fn_names = frozenset({"get_weather"})
130
+ assert is_function_recipient("unknown_tool", fn_names) is False
131
+
132
+ def test_dotted_name_accepted_when_in_allowed_names(self):
133
+ fn_names = frozenset({"math.sum", "namespace.tool_name"})
134
+ assert is_function_recipient("math.sum", fn_names) is True
135
+ assert is_function_recipient("namespace.tool_name", fn_names) is True
136
+
137
+ def test_dotted_name_rejected_when_not_in_allowed_names(self):
138
+ fn_names = frozenset({"get_weather"})
139
+ assert is_function_recipient("custom_server.search", fn_names) is False
140
+
141
+ def test_empty_allowed_names_rejects_bare_names(self):
142
+ """Empty frozenset means no function tools — bare names are not functions."""
143
+ fn_names: frozenset[str] = frozenset()
144
+ assert is_function_recipient("get_weather", fn_names) is False
145
+ assert is_function_recipient("math.sum", fn_names) is False
146
+
147
+ def test_builtin_tools_always_rejected(self):
148
+ fn_names = frozenset({"python", "browser", "container"})
149
+ assert is_function_recipient("python", fn_names) is False
150
+ assert is_function_recipient("browser", fn_names) is False
151
+ assert is_function_recipient("container", fn_names) is False
152
+
153
+ def test_builtin_dotted_always_rejected(self):
154
+ fn_names = frozenset({"python.run", "browser.search"})
155
+ assert is_function_recipient("python.run", fn_names) is False
156
+ assert is_function_recipient("browser.search", fn_names) is False
157
+
158
+ def test_none_allowed_names_uses_heuristic(self):
159
+ """When allowed names is None (Chat Completions), use heuristic."""
160
+ assert is_function_recipient("get_weather", None) is True
161
+ assert is_function_recipient("math.sum", None) is True
162
+ assert is_function_recipient("python", None) is False
163
+
164
+
165
+class TestExtractFunctionFromRecipient:
166
+ @pytest.mark.parametrize(
167
+ "recipient,expected",
168
+ [
169
+ ("functions.get_weather", "get_weather"),
170
+ ("functions.search_web", "search_web"),
171
+ ("functions.", ""),
172
+ ],
173
+ )
174
+ def test_strips_functions_prefix(self, recipient, expected):
175
+ assert extract_function_from_recipient(recipient) == expected
176
+
177
+ @pytest.mark.parametrize(
178
+ "recipient",
179
+ [
180
+ "get_weather",
181
+ "calculator",
182
+ "my-tool",
183
+ ],
184
+ )
185
+ def test_bare_name_returned_as_is(self, recipient):
186
+ assert extract_function_from_recipient(recipient) == recipient
187
+
188
+ @pytest.mark.parametrize(
189
+ "recipient,expected",
190
+ [
191
+ ("functions.math.sum", "math.sum"),
192
+ ("math.sum", "math.sum"),
193
+ ("namespace.tool_name", "namespace.tool_name"),
194
+ ],
195
+ )
196
+ def test_dotted_function_name_extraction(self, recipient, expected):
197
+ assert extract_function_from_recipient(recipient) == expected
198
+
199
+
200
class TestCommonParseInputToHarmonyMessage:
201
"""
202
Tests for scenarios that are common to both Chat Completion
tests/entrypoints/openai/responses/test_harmony_utils.py
+175
-5
@@ -246,7 +246,8 @@ class TestHarmonyToResponseOutput:
246
message = message.with_channel("commentary")
247
message = message.with_recipient("custom_tool")
248
249
- output_items = harmony_to_response_output(message)
249
+ fn_names = frozenset({"other_tool"})
250
+ output_items = harmony_to_response_output(message, fn_names)
251
252
assert len(output_items) == 1
253
assert isinstance(output_items[0], McpCall)
@@ -286,13 +287,179 @@ class TestHarmonyToResponseOutput:
287
assert len(output_items) == 0
288
289
290
+class TestHarmonyToResponseOutputWithFunctionToolNames:
291
+ """Tests for bare function name handling with function_tool_names."""
292
+
293
+ def test_bare_name_creates_function_call_when_in_tool_names(self):
294
+ """Bare function name matching a known tool creates function call."""
295
+ message = Message.from_role_and_content(
296
+ Role.ASSISTANT, '{"location": "San Francisco"}'
297
+ )
298
+ message = message.with_channel("commentary")
299
+ message = message.with_recipient("get_weather")
300
+
301
+ fn_names = frozenset({"get_weather"})
302
+ output_items = harmony_to_response_output(message, fn_names)
303
+
304
+ assert len(output_items) == 1
305
+ assert isinstance(output_items[0], ResponseFunctionToolCall)
306
+ assert output_items[0].type == "function_call"
307
+ assert output_items[0].name == "get_weather"
308
+ assert output_items[0].arguments == '{"location": "San Francisco"}'
309
+
310
+ def test_bare_name_creates_mcp_call_when_not_in_tool_names(self):
311
+ """Bare name not matching any known tool creates MCP call."""
312
+ message = Message.from_role_and_content(Role.ASSISTANT, '{"arg": "value"}')
313
+ message = message.with_channel("commentary")
314
+ message = message.with_recipient("custom_tool")
315
+
316
+ fn_names = frozenset({"get_weather"})
317
+ output_items = harmony_to_response_output(message, fn_names)
318
+
319
+ assert len(output_items) == 1
320
+ assert isinstance(output_items[0], McpCall)
321
+ assert output_items[0].type == "mcp_call"
322
+
323
+ def test_dotted_function_name_creates_function_call(self):
324
+ """Dotted function name in tool names creates function call."""
325
+ message = Message.from_role_and_content(Role.ASSISTANT, '{"a": 1, "b": 2}')
326
+ message = message.with_channel("commentary")
327
+ message = message.with_recipient("math.sum")
328
+
329
+ fn_names = frozenset({"math.sum"})
330
+ output_items = harmony_to_response_output(message, fn_names)
331
+
332
+ assert len(output_items) == 1
333
+ assert isinstance(output_items[0], ResponseFunctionToolCall)
334
+ assert output_items[0].name == "math.sum"
335
+
336
+ def test_empty_tool_names_defaults_to_mcp(self):
337
+ """With empty function_tool_names, bare names become MCP calls."""
338
+ message = Message.from_role_and_content(Role.ASSISTANT, '{"arg": "value"}')
339
+ message = message.with_channel("commentary")
340
+ message = message.with_recipient("get_weather")
341
+
342
+ output_items = harmony_to_response_output(message, frozenset())
343
+
344
+ assert len(output_items) == 1
345
+ assert isinstance(output_items[0], McpCall)
346
+
347
+ def test_prefixed_name_always_function_call(self):
348
+ """functions. prefix always creates function call even with empty tool names."""
349
+ message = Message.from_role_and_content(Role.ASSISTANT, '{"arg": "value"}')
350
+ message = message.with_channel("commentary")
351
+ message = message.with_recipient("functions.get_weather")
352
+
353
+ output_items = harmony_to_response_output(message, frozenset())
354
+
355
+ assert len(output_items) == 1
356
+ assert isinstance(output_items[0], ResponseFunctionToolCall)
357
+ assert output_items[0].name == "get_weather"
358
+
359
+
360
+class TestParserStateWithFunctionToolNames:
361
+ """Tests for parser_state_to_response_output with function_tool_names."""
362
+
363
+ def test_bare_name_creates_function_call(self):
364
+ from unittest.mock import Mock
365
+
366
+ parser = Mock()
367
+ parser.current_content = '{"arg": "value"}'
368
+ parser.current_role = Role.ASSISTANT
369
+ parser.current_channel = "commentary"
370
+ parser.current_recipient = "get_weather"
371
+
372
+ fn_names = frozenset({"get_weather"})
373
+ items = parser_state_to_response_output(parser, fn_names)
374
+
375
+ assert len(items) == 1
376
+ assert isinstance(items[0], ResponseFunctionToolCall)
377
+ assert items[0].name == "get_weather"
378
+ assert items[0].status == "in_progress"
379
+
380
+ def test_bare_name_creates_mcp_when_not_in_tool_names(self):
381
+ from unittest.mock import Mock
382
+
383
+ parser = Mock()
384
+ parser.current_content = '{"arg": "value"}'
385
+ parser.current_role = Role.ASSISTANT
386
+ parser.current_channel = "commentary"
387
+ parser.current_recipient = "unknown_tool"
388
+
389
+ fn_names = frozenset({"get_weather"})
390
+ items = parser_state_to_response_output(parser, fn_names)
391
+
392
+ assert len(items) == 1
393
+ assert isinstance(items[0], McpCall)
394
+ assert items[0].name == "unknown_tool"
395
+
396
+
397
+class TestToolCallsOnNonStandardChannels:
398
+ """Tests verifying tool calls are detected regardless of channel."""
399
+
400
+ def test_function_call_on_comment_channel(self):
401
+ message = Message.from_role_and_content(Role.ASSISTANT, '{"query": "weather"}')
402
+ message = message.with_channel("comment")
403
+ message = message.with_recipient("functions.get_weather")
404
+
405
+ output_items = harmony_to_response_output(message)
406
+
407
+ assert len(output_items) == 1
408
+ assert isinstance(output_items[0], ResponseFunctionToolCall)
409
+ assert output_items[0].type == "function_call"
410
+ assert output_items[0].name == "get_weather"
411
+
412
+ def test_bare_function_on_comment_channel(self):
413
+ message = Message.from_role_and_content(Role.ASSISTANT, '{"query": "weather"}')
414
+ message = message.with_channel("comment")
415
+ message = message.with_recipient("get_weather")
416
+
417
+ fn_names = frozenset({"get_weather"})
418
+ output_items = harmony_to_response_output(message, fn_names)
419
+
420
+ assert len(output_items) == 1
421
+ assert isinstance(output_items[0], ResponseFunctionToolCall)
422
+ assert output_items[0].name == "get_weather"
423
+
424
+ def test_parser_state_comment_channel_function(self):
425
+ from unittest.mock import Mock
426
+
427
+ parser = Mock()
428
+ parser.current_content = '{"arg": "value"}'
429
+ parser.current_role = Role.ASSISTANT
430
+ parser.current_channel = "comment"
431
+ parser.current_recipient = "functions.get_weather"
432
+
433
+ items = parser_state_to_response_output(parser)
434
+
435
+ assert len(items) == 1
436
+ assert isinstance(items[0], ResponseFunctionToolCall)
437
+ assert items[0].name == "get_weather"
438
+
439
+ def test_parser_state_comment_channel_mcp(self):
440
+ from unittest.mock import Mock
441
+
442
+ parser = Mock()
443
+ parser.current_content = '{"arg": "value"}'
444
+ parser.current_role = Role.ASSISTANT
445
+ parser.current_channel = "comment"
446
+ parser.current_recipient = "mcp.server.tool"
447
+
448
+ fn_names: frozenset[str] = frozenset()
449
+ items = parser_state_to_response_output(parser, fn_names)
450
+
451
+ assert len(items) == 1
452
+ assert isinstance(items[0], McpCall)
453
+
454
+
455
def test_parse_mcp_call_basic() -> None:
456
"""Test that MCP calls are parsed with correct type and server_label."""
457
message = Message.from_role_and_content(Role.ASSISTANT, '{"path": "/tmp"}')
458
message = message.with_recipient("filesystem")
459
message = message.with_channel("commentary")
460
295
- output_items = harmony_to_response_output(message)
461
+ fn_names: frozenset[str] = frozenset()
462
+ output_items = harmony_to_response_output(message, fn_names)
463
464
assert len(output_items) == 1
465
assert isinstance(output_items[0], McpCall)
@@ -309,7 +476,8 @@ def test_parse_mcp_call_dotted_recipient() -> None:
476
message = message.with_recipient("repo_browser.list")
477
message = message.with_channel("commentary")
478
312
- output_items = harmony_to_response_output(message)
479
+ fn_names: frozenset[str] = frozenset()
480
+ output_items = harmony_to_response_output(message, fn_names)
481
482
assert len(output_items) == 1
483
assert isinstance(output_items[0], McpCall)
@@ -371,7 +539,8 @@ def test_parser_state_to_response_output_commentary_channel() -> None:
539
parser_mcp.current_channel = "commentary"
540
parser_mcp.current_recipient = "filesystem"
541
374
- mcp_items = parser_state_to_response_output(parser_mcp)
542
+ fn_names: frozenset[str] = frozenset()
543
+ mcp_items = parser_state_to_response_output(parser_mcp, fn_names)
544
545
assert len(mcp_items) == 1
546
assert isinstance(mcp_items[0], McpCall)
@@ -438,7 +607,8 @@ def test_parser_state_to_response_output_analysis_channel() -> None:
607
parser_mcp.current_channel = "analysis"
608
parser_mcp.current_recipient = "database"
609
441
- mcp_items = parser_state_to_response_output(parser_mcp)
610
+ fn_names: frozenset[str] = frozenset()
611
+ mcp_items = parser_state_to_response_output(parser_mcp, fn_names)
612
613
assert len(mcp_items) == 1
614
assert isinstance(mcp_items[0], McpCall)
tests/tool_parsers/test_openai_tool_parser.py
+152
@@ -217,6 +217,158 @@ def test_extract_tool_calls_multiple_tools(
217
assert extracted_info.content is None
218
219
220
+def test_extract_tool_calls_bare_function_name(
221
+ openai_tool_parser,
222
+ harmony_encoding,
223
+):
224
+ convo = Conversation.from_messages(
225
+ [
226
+ Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?"),
227
+ Message.from_role_and_content(
228
+ Role.ASSISTANT,
229
+ "We need to use get_current_weather tool.",
230
+ ).with_channel("analysis"),
231
+ Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
232
+ .with_channel("commentary")
233
+ .with_recipient("get_current_weather")
234
+ .with_content_type("json"),
235
+ ]
236
+ )
237
+ token_ids = harmony_encoding.render_conversation_for_completion(
238
+ convo, Role.ASSISTANT
239
+ )
240
+
241
+ extracted_info = openai_tool_parser.extract_tool_calls(
242
+ "",
243
+ request=None,
244
+ token_ids=token_ids,
245
+ )
246
+ assert extracted_info.tools_called
247
+ expected_tool_calls = [
248
+ ToolCall(
249
+ function=FunctionCall(
250
+ name="get_current_weather",
251
+ arguments=json.dumps({"location": "Tokyo"}),
252
+ )
253
+ )
254
+ ]
255
+ assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
256
+ assert extracted_info.content is None
257
+
258
+
259
+def test_extract_tool_calls_bare_function_name_multiple(
260
+ openai_tool_parser,
261
+ harmony_encoding,
262
+):
263
+ convo = Conversation.from_messages(
264
+ [
265
+ Message.from_role_and_content(
266
+ Role.USER, "What is the weather in Tokyo based on where I'm at?"
267
+ ),
268
+ Message.from_role_and_content(
269
+ Role.ASSISTANT,
270
+ "We need to use both tools.",
271
+ ).with_channel("analysis"),
272
+ Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
273
+ .with_channel("commentary")
274
+ .with_recipient("get_current_weather")
275
+ .with_content_type("json"),
276
+ Message.from_role_and_content(Role.ASSISTANT, "{}")
277
+ .with_channel("commentary")
278
+ .with_recipient("get_user_location")
279
+ .with_content_type("json"),
280
+ ]
281
+ )
282
+ token_ids = harmony_encoding.render_conversation_for_completion(
283
+ convo, Role.ASSISTANT
284
+ )
285
+
286
+ extracted_info = openai_tool_parser.extract_tool_calls(
287
+ "",
288
+ request=None,
289
+ token_ids=token_ids,
290
+ )
291
+ assert extracted_info.tools_called
292
+ expected_tool_calls = [
293
+ ToolCall(
294
+ function=FunctionCall(
295
+ name="get_current_weather",
296
+ arguments=json.dumps({"location": "Tokyo"}),
297
+ )
298
+ ),
299
+ ToolCall(
300
+ function=FunctionCall(
301
+ name="get_user_location",
302
+ arguments=json.dumps({}),
303
+ )
304
+ ),
305
+ ]
306
+ assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
307
+
308
+
309
+def test_extract_tool_calls_assistant_recipient_ignored(
310
+ openai_tool_parser,
311
+ harmony_encoding,
312
+):
313
+ convo = Conversation.from_messages(
314
+ [
315
+ Message.from_role_and_content(Role.USER, "Hello"),
316
+ Message.from_role_and_content(Role.ASSISTANT, "Some tool response")
317
+ .with_channel("commentary")
318
+ .with_recipient("assistant"),
319
+ Message.from_role_and_content(
320
+ Role.ASSISTANT, "Here is the answer"
321
+ ).with_channel("final"),
322
+ ]
323
+ )
324
+ token_ids = harmony_encoding.render_conversation_for_completion(
325
+ convo, Role.ASSISTANT
326
+ )
327
+
328
+ extracted_info = openai_tool_parser.extract_tool_calls(
329
+ "",
330
+ request=None,
331
+ token_ids=token_ids,
332
+ )
333
+ assert not extracted_info.tools_called
334
+ assert extracted_info.tool_calls == []
335
+ assert extracted_info.content == "Here is the answer"
336
+
337
+
338
+def test_extract_tool_calls_dotted_function_name(
339
+ openai_tool_parser,
340
+ harmony_encoding,
341
+):
342
+ convo = Conversation.from_messages(
343
+ [
344
+ Message.from_role_and_content(Role.USER, "Compute 2+3"),
345
+ Message.from_role_and_content(Role.ASSISTANT, '{"a": 2, "b": 3}')
346
+ .with_channel("commentary")
347
+ .with_recipient("math.sum")
348
+ .with_content_type("json"),
349
+ ]
350
+ )
351
+ token_ids = harmony_encoding.render_conversation_for_completion(
352
+ convo, Role.ASSISTANT
353
+ )
354
+
355
+ extracted_info = openai_tool_parser.extract_tool_calls(
356
+ "",
357
+ request=None,
358
+ token_ids=token_ids,
359
+ )
360
+ assert extracted_info.tools_called
361
+ expected_tool_calls = [
362
+ ToolCall(
363
+ function=FunctionCall(
364
+ name="math.sum",
365
+ arguments=json.dumps({"a": 2, "b": 3}),
366
+ )
367
+ )
368
+ ]
369
+ assert_tool_calls(extracted_info.tool_calls, expected_tool_calls)
370
+
371
+
372
def test_extract_tool_calls_with_content(
373
openai_tool_parser,
374
harmony_encoding,
vllm/entrypoints/openai/chat_completion/stream_harmony.py
+8
-12
@@ -17,6 +17,10 @@ from vllm.entrypoints.openai.engine.protocol import (
17
DeltaMessage,
18
DeltaToolCall,
19
)
20
+from vllm.entrypoints.openai.parser.harmony_utils import (
21
+ extract_function_from_recipient,
22
+ is_function_recipient,
23
+)
24
25
26
class TokenState(NamedTuple):
@@ -79,16 +83,12 @@ def extract_harmony_streaming_delta(
83
# This counts completed tool calls in messages
84
base_index = 0
85
for msg in harmony_parser.messages:
82
- if (
83
- (msg.channel == "commentary" or msg.channel == "analysis")
84
- and msg.recipient
85
- and msg.recipient.startswith("functions.")
86
- ):
86
+ if msg.recipient and is_function_recipient(msg.recipient):
87
base_index += 1
88
89
# If there's an ongoing tool call from previous chunk,
90
# the next new tool call starts at base_index + 1
91
- if prev_recipient and prev_recipient.startswith("functions."):
91
+ if prev_recipient and is_function_recipient(prev_recipient):
92
next_tool_index = base_index + 1
93
# Ongoing call is at base_index
94
ongoing_tool_index = base_index
@@ -101,15 +101,11 @@ def extract_harmony_streaming_delta(
101
if group.channel == "final":
102
combined_content += group.text
103
content_encountered = True
104
- elif (
105
- (group.channel == "commentary" or group.channel == "analysis")
106
- and group.recipient
107
- and group.recipient.startswith("functions.")
108
- ):
104
+ elif group.recipient and is_function_recipient(group.recipient):
105
opened_new_call = False
106
if prev_recipient != group.recipient:
107
# New tool call - emit the opening message
112
- tool_name = group.recipient.split("functions.", 1)[1]
108
+ tool_name = extract_function_from_recipient(group.recipient)
109
tool_messages.append(
110
DeltaToolCall(
111
id=make_tool_call_id(),
vllm/entrypoints/openai/parser/harmony_utils.py
+36
@@ -26,6 +26,42 @@ from vllm.logger import init_logger
26
27
logger = init_logger(__name__)
28
29
+
30
+def is_function_recipient(
31
+ recipient: str,
32
+ allowed_function_tool_names: frozenset[str] | None = None,
33
+) -> bool:
34
+ """Check whether *recipient* refers to a function tool call.
35
+
36
+ The optional *allowed_function_tool_names* parameter is used by the
37
+ Responses API to distinguish bare function-call recipients (missing the
38
+ ``functions.`` prefix) from MCP tool calls. When provided, a bare
39
+ recipient is only treated as a function call if it appears in the set.
40
+ The Chat Completions path omits this parameter so that all bare
41
+ recipients are accepted as function calls (the heuristic fallback).
42
+ """
43
+ if not recipient:
44
+ return False
45
+ if recipient.startswith("<|"):
46
+ return False
47
+ if recipient.startswith("functions."):
48
+ return len(recipient) > len("functions.")
49
+ if recipient == "assistant":
50
+ return False
51
+ if recipient in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
52
+ return False
53
+ first_segment = recipient.split(".", 1)[0]
54
+ if first_segment in BUILTIN_TOOL_TO_MCP_SERVER_LABEL:
55
+ return False
56
+ if allowed_function_tool_names is not None:
57
+ return recipient in allowed_function_tool_names
58
+ return True
59
+
60
+
61
+def extract_function_from_recipient(recipient: str) -> str:
62
+ return recipient.removeprefix("functions.")
63
+
64
+
65
REASONING_EFFORT = {
66
"high": ReasoningEffort.HIGH,
67
"medium": ReasoningEffort.MEDIUM,
vllm/entrypoints/openai/responses/context.py
+2
@@ -524,10 +524,12 @@ class HarmonyContext(ConversationContext):
524
self,
525
messages: list,
526
available_tools: list[str],
527
+ function_tool_names: frozenset[str] | None = None,
528
):
529
self._messages = messages
530
self.finish_reason: str | None = None
531
self.available_tools = available_tools
532
+ self.function_tool_names = function_tool_names
533
self._tool_sessions: dict[str, ClientSession | Tool] = {}
534
self.called_tools: set[str] = set()
535
vllm/entrypoints/openai/responses/harmony.py
+13
-7
@@ -31,7 +31,9 @@ from openai_harmony import Author, Message, Role, StreamableParser, TextContent
31
32
from vllm.entrypoints.openai.parser.harmony_utils import (
33
BUILTIN_TOOL_TO_MCP_SERVER_LABEL,
34
+ extract_function_from_recipient,
35
flatten_chat_text_content,
36
+ is_function_recipient,
37
)
38
from vllm.entrypoints.openai.responses.protocol import (
39
ResponseInputOutputItem,
@@ -294,7 +296,7 @@ def _parse_browser_tool_call(message: Message, recipient: str) -> ResponseOutput
296
297
def _parse_function_call(message: Message, recipient: str) -> list[ResponseOutputItem]:
298
"""Parse function calls into function tool call items."""
297
- function_name = recipient.split(".")[-1]
299
+ function_name = extract_function_from_recipient(recipient)
300
output_items = []
301
for content in message.content:
302
random_id = random_uuid()
@@ -410,7 +412,10 @@ def _parse_message_no_recipient(
412
# ---------------------------------------------------------------------------
413
414
413
-def harmony_to_response_output(message: Message) -> list[ResponseOutputItem]:
415
+def harmony_to_response_output(
416
+ message: Message,
417
+ function_tool_names: frozenset[str] | None = None,
418
+) -> list[ResponseOutputItem]:
419
"""Parse a Harmony message into a list of output response items.
420
421
This is the main dispatcher that routes based on channel and recipient.
@@ -429,8 +434,8 @@ def harmony_to_response_output(message: Message) -> list[ResponseOutputItem]:
434
if recipient.startswith("browser."):
435
output_items.append(_parse_browser_tool_call(message, recipient))
436
432
- # Function calls (should only happen on commentary channel)
433
- elif message.channel == "commentary" and recipient.startswith("functions."):
437
+ # Function calls (with or without "functions." prefix)
438
+ elif is_function_recipient(recipient, function_tool_names):
439
output_items.extend(_parse_function_call(message, recipient))
440
441
# Built-in MCP tools (python, browser, container)
@@ -450,6 +455,7 @@ def harmony_to_response_output(message: Message) -> list[ResponseOutputItem]:
455
456
def parser_state_to_response_output(
457
parser: StreamableParser,
458
+ function_tool_names: frozenset[str] | None = None,
459
) -> list[ResponseOutputItem]:
460
"""Extract in-progress response items from incomplete parser state.
461
@@ -464,15 +470,15 @@ def parser_state_to_response_output(
470
if current_recipient is not None and current_recipient.startswith("browser."):
471
return []
472
467
- if current_recipient and parser.current_channel in ("commentary", "analysis"):
468
- if current_recipient.startswith("functions."):
473
+ if current_recipient:
474
+ if is_function_recipient(current_recipient, function_tool_names):
475
rid = random_uuid()
476
return [
477
ResponseFunctionToolCall(
478
arguments=parser.current_content,
479
call_id=f"call_{rid}",
480
type="function_call",
475
- name=current_recipient.split(".")[-1],
481
+ name=extract_function_from_recipient(current_recipient),
482
id=f"fc_{rid}",
483
status="in_progress",
484
)
vllm/entrypoints/openai/responses/serving.py
+14
-5
@@ -89,6 +89,7 @@ from vllm.entrypoints.openai.responses.streaming_events import (
89
from vllm.entrypoints.openai.responses.utils import (
90
construct_input_messages,
91
construct_tool_dicts,
92
+ extract_function_tool_names,
93
extract_tool_types,
94
)
95
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
@@ -449,11 +450,16 @@ class OpenAIServingResponses(OpenAIServing):
450
)
451
452
context: ConversationContext
453
+ function_tool_names = extract_function_tool_names(request.tools)
454
if self.use_harmony:
455
if request.stream:
454
- context = StreamingHarmonyContext(messages, available_tools)
456
+ context = StreamingHarmonyContext(
457
+ messages, available_tools, function_tool_names
458
+ )
459
else:
456
- context = HarmonyContext(messages, available_tools)
460
+ context = HarmonyContext(
461
+ messages, available_tools, function_tool_names
462
+ )
463
else:
464
if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
465
# This is a feature in development for parsing
@@ -1070,10 +1076,11 @@ class OpenAIServingResponses(OpenAIServing):
1076
) -> list[ResponseOutputItem]:
1077
output_items: list[ResponseOutputItem] = []
1078
num_init_messages = context.num_init_messages
1079
+ fn_names = context.function_tool_names
1080
for msg in context.messages[num_init_messages:]:
1074
- output_items.extend(harmony_to_response_output(msg))
1081
+ output_items.extend(harmony_to_response_output(msg, fn_names))
1082
# Handle the generation stopped in the middle (if any).
1076
- last_items = parser_state_to_response_output(context.parser)
1083
+ last_items = parser_state_to_response_output(context.parser, fn_names)
1084
if last_items:
1085
output_items.extend(last_items)
1086
return output_items
@@ -1448,7 +1455,9 @@ class OpenAIServingResponses(OpenAIServing):
1455
if ctx.is_expecting_start():
1456
if len(ctx.parser.messages) > 0:
1457
previous_item = ctx.parser.messages[-1]
1451
- for event in emit_previous_item_done_events(previous_item, state):
1458
+ for event in emit_previous_item_done_events(
1459
+ previous_item, state, ctx.function_tool_names
1460
+ ):
1461
yield _increment_sequence_number_and_return(event)
1462
state.reset_for_new_item()
1463
vllm/entrypoints/openai/responses/streaming_events.py
+26
-18
@@ -62,6 +62,10 @@ from openai_harmony import Message as HarmonyMessage
62
63
from vllm.entrypoints.mcp.tool_server import ToolServer
64
from vllm.entrypoints.openai.engine.protocol import DeltaMessage
65
+from vllm.entrypoints.openai.parser.harmony_utils import (
66
+ extract_function_from_recipient,
67
+ is_function_recipient,
68
+)
69
from vllm.entrypoints.openai.responses.context import StreamingHarmonyContext
70
from vllm.entrypoints.openai.responses.protocol import (
71
ResponseReasoningPartAddedEvent,
@@ -111,19 +115,19 @@ class StreamingState:
115
self.current_call_id = ""
116
117
114
-def is_mcp_tool_by_namespace(recipient: str | None) -> bool:
118
+def is_mcp_tool_by_namespace(
119
+ recipient: str | None,
120
+ allowed_function_tool_names: frozenset[str] | None = None,
121
+) -> bool:
122
"""
123
Determine if a tool call is an MCP tool based on recipient prefix.
124
118
- - Tools starting with "functions." are function calls
119
- - Everything else is an MCP tool
125
+ Inverse of :func:`is_function_recipient` — everything that is not
126
+ a function call is an MCP tool.
127
"""
128
if recipient is None:
129
return False
123
-
124
- # Function calls have "functions." prefix
125
- # Everything else is an MCP tool
126
- return not recipient.startswith("functions.")
130
+ return not is_function_recipient(recipient, allowed_function_tool_names)
131
132
133
# =====================================================================
@@ -575,16 +579,16 @@ def emit_content_delta_events(
579
return emit_text_delta_events(delta, state)
580
elif channel == "analysis" and recipient is None:
581
return emit_reasoning_delta_events(delta, state)
578
- # built-in tools will be triggered on the analysis channel
579
- # However, occasionally built-in tools will
580
- # still be output to commentary.
581
- elif channel in ("commentary", "analysis") and recipient is not None:
582
- if recipient.startswith("functions."):
583
- function_name = recipient[len("functions.") :]
582
+ elif recipient is not None:
583
+ fn_names = ctx.function_tool_names
584
+ if is_function_recipient(recipient, fn_names):
585
+ function_name = extract_function_from_recipient(recipient)
586
return emit_function_call_delta_events(delta, function_name, state)
587
elif recipient == "python":
588
return emit_code_interpreter_delta_events(delta, state)
587
- elif recipient.startswith("mcp.") or is_mcp_tool_by_namespace(recipient):
589
+ elif recipient.startswith("mcp.") or is_mcp_tool_by_namespace(
590
+ recipient, fn_names
591
+ ):
592
return emit_mcp_delta_events(delta, state, recipient)
593
594
return []
@@ -593,6 +597,7 @@ def emit_content_delta_events(
597
def emit_previous_item_done_events(
598
previous_item: HarmonyMessage,
599
state: StreamingState,
600
+ function_tool_names: frozenset[str] | None = None,
601
) -> list[StreamingResponsesResponse]:
602
"""Emit done events for the previous item when expecting a new start.
603
@@ -602,13 +607,13 @@ def emit_previous_item_done_events(
607
text = previous_item.content[0].text
608
if previous_item.recipient is not None:
609
# Deal with tool call
605
- if previous_item.recipient.startswith("functions."):
606
- function_name = previous_item.recipient[len("functions.") :]
610
+ if is_function_recipient(previous_item.recipient, function_tool_names):
611
+ function_name = extract_function_from_recipient(previous_item.recipient)
612
return emit_function_call_done_events(function_name, text, state)
613
elif previous_item.recipient == "python":
614
return emit_code_interpreter_completion_events(previous_item, state)
615
elif (
611
- is_mcp_tool_by_namespace(previous_item.recipient)
616
+ is_mcp_tool_by_namespace(previous_item.recipient, function_tool_names)
617
and state.current_item_id is not None
618
and state.current_item_id.startswith("mcp_")
619
):
@@ -792,9 +797,12 @@ def emit_tool_action_events(
797
and state.sent_output_item_added
798
):
799
recipient = previous_item.recipient
800
+ fn_names = ctx.function_tool_names
801
if recipient == "python":
802
events.extend(emit_code_interpreter_completion_events(previous_item, state))
797
- elif recipient.startswith("mcp.") or is_mcp_tool_by_namespace(recipient):
803
+ elif recipient.startswith("mcp.") or is_mcp_tool_by_namespace(
804
+ recipient, fn_names
805
+ ):
806
events.extend(
807
emit_mcp_completion_events(
808
recipient, previous_item.content[0].text, state
vllm/entrypoints/openai/responses/utils.py
+4
@@ -236,6 +236,10 @@ def _construct_message_from_response_item(
236
return item # type: ignore[arg-type]
237
238
239
+def extract_function_tool_names(tools: list[Tool]) -> frozenset[str]:
240
+ return frozenset(tool.name for tool in tools if tool.type == "function")
241
+
242
+
243
def extract_tool_types(tools: list[Tool]) -> set[str]:
244
"""
245
Extracts the tool types from the given tools.
vllm/tool_parsers/openai_tool_parser.py
+9
-3
@@ -13,7 +13,11 @@ from vllm.entrypoints.openai.engine.protocol import (
13
FunctionCall,
14
ToolCall,
15
)
16
-from vllm.entrypoints.openai.parser.harmony_utils import parse_output_into_messages
16
+from vllm.entrypoints.openai.parser.harmony_utils import (
17
+ extract_function_from_recipient,
18
+ is_function_recipient,
19
+ parse_output_into_messages,
20
+)
21
from vllm.logger import init_logger
22
from vllm.tool_parsers.abstract_tool_parser import (
23
Tool,
@@ -50,10 +54,12 @@ class OpenAIToolParser(ToolParser):
54
55
if len(parser.messages) > 0:
56
for msg in parser.messages:
57
+ if msg.author.role != "assistant":
58
+ continue
59
if len(msg.content) < 1:
60
continue
61
msg_text = msg.content[0].text
56
- if msg.recipient and msg.recipient.startswith("functions."):
62
+ if msg.recipient and is_function_recipient(msg.recipient):
63
# If no content-type is given assume JSON, as that's the
64
# most common case with gpt-oss models.
65
if not msg.content_type or "json" in msg.content_type:
@@ -72,7 +78,7 @@ class OpenAIToolParser(ToolParser):
78
ToolCall(
79
type="function",
80
function=FunctionCall(
75
- name=msg.recipient.split("functions.")[1],
81
+ name=extract_function_from_recipient(msg.recipient),
82
arguments=tool_args,
83
),
84
)