A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 21:15 UTC
[Frontend] Add MCP type support infrastructure to Responses API (#30054)
Signed-off-by: Daniel Salib <danielsalib@meta.com>
daniel-salib committed
Dec 7, 2025 at 18:02 UTC
444f0e3f339caba85f84c6628e1df50605b241a0
3 files changed
+309
-47
tests/entrypoints/openai/parser/test_harmony_utils.py
+176
-9
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
from openai.types.responses import ResponseFunctionToolCall, ResponseReasoningItem
5
+from openai.types.responses.response_output_item import McpCall
6
from openai_harmony import Author, Message, Role, TextContent
7
8
from vllm.entrypoints.openai.parser.harmony_utils import (
@@ -400,17 +401,19 @@ class TestParseOutputMessage:
401
assert output_items[0].arguments == '{"location": "San Francisco"}'
402
assert output_items[1].arguments == '{"location": "New York"}'
403
403
- def test_commentary_with_unknown_recipient_raises_error(self):
404
- """Test that commentary with unknown recipient raises ValueError."""
405
- message = Message.from_role_and_content(Role.ASSISTANT, "some content")
404
+ def test_commentary_with_unknown_recipient_creates_mcp_call(self):
405
+ """Test that commentary with unknown recipient creates MCP call."""
406
+ message = Message.from_role_and_content(Role.ASSISTANT, '{"arg": "value"}')
407
message = message.with_channel("commentary")
407
- message = message.with_recipient("unknown_recipient")
408
+ message = message.with_recipient("custom_tool")
409
409
- try:
410
- parse_output_message(message)
411
- raise AssertionError("Expected ValueError to be raised")
412
- except ValueError as e:
413
- assert "Unknown recipient: unknown_recipient" in str(e)
410
+ output_items = parse_output_message(message)
411
+
412
+ assert len(output_items) == 1
413
+ assert isinstance(output_items[0], McpCall)
414
+ assert output_items[0].type == "mcp_call"
415
+ assert output_items[0].name == "custom_tool"
416
+ assert output_items[0].server_label == "custom_tool"
417
418
def test_analysis_channel_creates_reasoning(self):
419
"""Test that analysis channel creates reasoning items."""
@@ -451,3 +454,167 @@ def test_has_custom_tools() -> None:
454
assert has_custom_tools(
455
{"web_search_preview", "code_interpreter", "container", "others"}
456
)
457
+
458
+
459
+def test_parse_mcp_call_basic() -> None:
460
+ """Test that MCP calls are parsed with correct type and server_label."""
461
+ message = Message.from_role_and_content(Role.ASSISTANT, '{"path": "/tmp"}')
462
+ message = message.with_recipient("filesystem")
463
+ message = message.with_channel("commentary")
464
+
465
+ output_items = parse_output_message(message)
466
+
467
+ assert len(output_items) == 1
468
+ assert isinstance(output_items[0], McpCall)
469
+ assert output_items[0].type == "mcp_call"
470
+ assert output_items[0].name == "filesystem"
471
+ assert output_items[0].server_label == "filesystem"
472
+ assert output_items[0].arguments == '{"path": "/tmp"}'
473
+ assert output_items[0].status == "completed"
474
+
475
+
476
+def test_parse_mcp_call_dotted_recipient() -> None:
477
+ """Test that dotted recipients extract the tool name correctly."""
478
+ message = Message.from_role_and_content(Role.ASSISTANT, '{"cmd": "ls"}')
479
+ message = message.with_recipient("repo_browser.list")
480
+ message = message.with_channel("commentary")
481
+
482
+ output_items = parse_output_message(message)
483
+
484
+ assert len(output_items) == 1
485
+ assert isinstance(output_items[0], McpCall)
486
+ assert output_items[0].name == "list"
487
+ assert output_items[0].server_label == "repo_browser"
488
+
489
+
490
+def test_mcp_vs_function_call() -> None:
491
+ """Test that function calls are not parsed as MCP calls."""
492
+ func_message = Message.from_role_and_content(Role.ASSISTANT, '{"arg": "value"}')
493
+ func_message = func_message.with_recipient("functions.my_tool")
494
+ func_message = func_message.with_channel("commentary")
495
+
496
+ func_items = parse_output_message(func_message)
497
+
498
+ assert len(func_items) == 1
499
+ assert not isinstance(func_items[0], McpCall)
500
+ assert func_items[0].type == "function_call"
501
+
502
+
503
+def test_mcp_vs_builtin_tools() -> None:
504
+ """Test that built-in tools (python, container) are not parsed as MCP calls."""
505
+ # Test python (built-in tool) - should be reasoning, not MCP
506
+ python_message = Message.from_role_and_content(Role.ASSISTANT, "print('hello')")
507
+ python_message = python_message.with_recipient("python")
508
+ python_message = python_message.with_channel("commentary")
509
+
510
+ python_items = parse_output_message(python_message)
511
+
512
+ assert len(python_items) == 1
513
+ assert not isinstance(python_items[0], McpCall)
514
+ assert python_items[0].type == "reasoning"
515
+
516
+
517
+def test_parse_remaining_state_commentary_channel() -> None:
518
+ """Test parse_remaining_state with commentary channel and various recipients."""
519
+ from unittest.mock import Mock
520
+
521
+ from vllm.entrypoints.openai.parser.harmony_utils import parse_remaining_state
522
+
523
+ # Test 1: functions.* recipient → should return function tool call
524
+ parser_func = Mock()
525
+ parser_func.current_content = '{"arg": "value"}'
526
+ parser_func.current_role = Role.ASSISTANT
527
+ parser_func.current_channel = "commentary"
528
+ parser_func.current_recipient = "functions.my_tool"
529
+
530
+ func_items = parse_remaining_state(parser_func)
531
+
532
+ assert len(func_items) == 1
533
+ assert not isinstance(func_items[0], McpCall)
534
+ assert func_items[0].type == "function_call"
535
+ assert func_items[0].name == "my_tool"
536
+ assert func_items[0].status == "in_progress"
537
+
538
+ # Test 2: MCP tool (not builtin) → should return MCP call
539
+ parser_mcp = Mock()
540
+ parser_mcp.current_content = '{"path": "/tmp"}'
541
+ parser_mcp.current_role = Role.ASSISTANT
542
+ parser_mcp.current_channel = "commentary"
543
+ parser_mcp.current_recipient = "filesystem"
544
+
545
+ mcp_items = parse_remaining_state(parser_mcp)
546
+
547
+ assert len(mcp_items) == 1
548
+ assert isinstance(mcp_items[0], McpCall)
549
+ assert mcp_items[0].type == "mcp_call"
550
+ assert mcp_items[0].name == "filesystem"
551
+ assert mcp_items[0].server_label == "filesystem"
552
+ assert mcp_items[0].status == "in_progress"
553
+
554
+ # Test 3: Built-in tool (python)
555
+ # should NOT return MCP call, falls through to reasoning
556
+ parser_builtin = Mock()
557
+ parser_builtin.current_content = "print('hello')"
558
+ parser_builtin.current_role = Role.ASSISTANT
559
+ parser_builtin.current_channel = "commentary"
560
+ parser_builtin.current_recipient = "python"
561
+
562
+ builtin_items = parse_remaining_state(parser_builtin)
563
+
564
+ # Should fall through to reasoning logic
565
+ assert len(builtin_items) == 1
566
+ assert not isinstance(builtin_items[0], McpCall)
567
+ assert builtin_items[0].type == "reasoning"
568
+
569
+
570
+def test_parse_remaining_state_analysis_channel() -> None:
571
+ """Test parse_remaining_state with analysis channel and various recipients."""
572
+ from unittest.mock import Mock
573
+
574
+ from vllm.entrypoints.openai.parser.harmony_utils import parse_remaining_state
575
+
576
+ # Test 1: functions.* recipient → should return function tool call
577
+ parser_func = Mock()
578
+ parser_func.current_content = '{"arg": "value"}'
579
+ parser_func.current_role = Role.ASSISTANT
580
+ parser_func.current_channel = "analysis"
581
+ parser_func.current_recipient = "functions.my_tool"
582
+
583
+ func_items = parse_remaining_state(parser_func)
584
+
585
+ assert len(func_items) == 1
586
+ assert not isinstance(func_items[0], McpCall)
587
+ assert func_items[0].type == "function_call"
588
+ assert func_items[0].name == "my_tool"
589
+ assert func_items[0].status == "in_progress"
590
+
591
+ # Test 2: MCP tool (not builtin) → should return MCP call
592
+ parser_mcp = Mock()
593
+ parser_mcp.current_content = '{"query": "test"}'
594
+ parser_mcp.current_role = Role.ASSISTANT
595
+ parser_mcp.current_channel = "analysis"
596
+ parser_mcp.current_recipient = "database"
597
+
598
+ mcp_items = parse_remaining_state(parser_mcp)
599
+
600
+ assert len(mcp_items) == 1
601
+ assert isinstance(mcp_items[0], McpCall)
602
+ assert mcp_items[0].type == "mcp_call"
603
+ assert mcp_items[0].name == "database"
604
+ assert mcp_items[0].server_label == "database"
605
+ assert mcp_items[0].status == "in_progress"
606
+
607
+ # Test 3: Built-in tool (container)
608
+ # should NOT return MCP call, falls through to reasoning
609
+ parser_builtin = Mock()
610
+ parser_builtin.current_content = "docker run"
611
+ parser_builtin.current_role = Role.ASSISTANT
612
+ parser_builtin.current_channel = "analysis"
613
+ parser_builtin.current_recipient = "container"
614
+
615
+ builtin_items = parse_remaining_state(parser_builtin)
616
+
617
+ # Should fall through to reasoning logic
618
+ assert len(builtin_items) == 1
619
+ assert not isinstance(builtin_items[0], McpCall)
620
+ assert builtin_items[0].type == "reasoning"
vllm/entrypoints/openai/parser/harmony_utils.py
+125
-38
@@ -19,6 +19,7 @@ from openai.types.responses.response_function_web_search import (
19
ActionSearch,
20
ResponseFunctionWebSearch,
21
)
22
+from openai.types.responses.response_output_item import McpCall
23
from openai.types.responses.response_reasoning_item import (
24
Content as ResponseReasoningTextContent,
25
)
@@ -155,11 +156,7 @@ def get_developer_message(
156
"web_search_preview",
157
"code_interpreter",
158
"container",
158
- "mcp",
159
):
160
- # These are built-in tools that are added to the system message.
161
- # Adding in MCP for now until we support MCP tools executed
162
- # server side
160
pass
161
162
elif tool.type == "function":
@@ -427,6 +424,44 @@ def _parse_final_message(message: Message) -> ResponseOutputItem:
424
)
425
426
427
+def _parse_mcp_recipient(recipient: str) -> tuple[str, str]:
428
+ """
429
+ Parse MCP recipient into (server_label, tool_name).
430
+
431
+ For dotted recipients like "repo_browser.list":
432
+ - server_label: "repo_browser" (namespace/server)
433
+ - tool_name: "list" (specific tool)
434
+
435
+ For simple recipients like "filesystem":
436
+ - server_label: "filesystem"
437
+ - tool_name: "filesystem"
438
+ """
439
+ if "." in recipient:
440
+ server_label = recipient.split(".")[0]
441
+ tool_name = recipient.split(".")[-1]
442
+ else:
443
+ server_label = recipient
444
+ tool_name = recipient
445
+ return server_label, tool_name
446
+
447
+
448
+def _parse_mcp_call(message: Message, recipient: str) -> list[ResponseOutputItem]:
449
+ """Parse MCP calls into MCP call items."""
450
+ server_label, tool_name = _parse_mcp_recipient(recipient)
451
+ output_items = []
452
+ for content in message.content:
453
+ response_item = McpCall(
454
+ arguments=content.text,
455
+ type="mcp_call",
456
+ name=tool_name,
457
+ server_label=server_label,
458
+ id=f"mcp_{random_uuid()}",
459
+ status="completed",
460
+ )
461
+ output_items.append(response_item)
462
+ return output_items
463
+
464
+
465
def parse_output_message(message: Message) -> list[ResponseOutputItem]:
466
"""
467
Parse a Harmony message into a list of output response items.
@@ -440,33 +475,34 @@ def parse_output_message(message: Message) -> list[ResponseOutputItem]:
475
output_items: list[ResponseOutputItem] = []
476
recipient = message.recipient
477
443
- # Browser tool calls
444
- if recipient is not None and recipient.startswith("browser."):
445
- output_items.append(_parse_browser_tool_call(message, recipient))
446
-
447
- # Analysis channel (reasoning/chain-of-thought)
448
- elif message.channel == "analysis":
449
- output_items.extend(_parse_reasoning_content(message))
478
+ if recipient is not None:
479
+ # Browser tool calls
480
+ if recipient.startswith("browser."):
481
+ output_items.append(_parse_browser_tool_call(message, recipient))
482
451
- # Commentary channel
452
- elif message.channel == "commentary":
453
- # Function calls
454
- if recipient is not None and recipient.startswith("functions."):
483
+ # Function calls (should only happen on commentary channel)
484
+ elif message.channel == "commentary" and recipient.startswith("functions."):
485
output_items.extend(_parse_function_call(message, recipient))
486
457
- # Built-in tools on commentary channel are treated as reasoning for now
458
- elif (
459
- recipient is None # Preambles: explanatory text before tool calls
460
- or recipient.startswith(("python", "browser", "container"))
461
- ):
462
- # Per Harmony format, commentary channel can contain preambles to calling
463
- # multiple functions - explanatory text with no recipient. Built-in tool
464
- # recipients (python/browser/container) also generate reasoning output.
487
+ # Built-in tools are treated as reasoning
488
+ elif recipient.startswith(("python", "browser", "container")):
489
+ # Built-in tool recipients (python/browser/container)
490
+ # generate reasoning output
491
output_items.extend(_parse_reasoning_content(message))
492
+
493
+ # All other recipients are MCP calls
494
else:
467
- raise ValueError(f"Unknown recipient: {recipient}")
495
+ output_items.extend(_parse_mcp_call(message, recipient))
496
+
497
+ # No recipient - handle based on channel for non-tool messages
498
+ elif message.channel == "analysis":
499
+ output_items.extend(_parse_reasoning_content(message))
500
+
501
+ elif message.channel == "commentary":
502
+ # Per Harmony format, commentary channel can contain preambles to calling
503
+ # multiple functions - explanatory text with no recipient
504
+ output_items.extend(_parse_reasoning_content(message))
505
469
- # Final output message
506
elif message.channel == "final":
507
output_items.append(_parse_final_message(message))
508
@@ -485,20 +521,70 @@ def parse_remaining_state(parser: StreamableParser) -> list[ResponseOutputItem]:
521
if current_recipient is not None and current_recipient.startswith("browser."):
522
return []
523
488
- if parser.current_channel == "analysis":
489
- reasoning_item = ResponseReasoningItem(
490
- id=f"rs_{random_uuid()}",
491
- summary=[],
492
- type="reasoning",
493
- content=[
494
- ResponseReasoningTextContent(
495
- text=parser.current_content, type="reasoning_text"
524
+ if current_recipient and parser.current_channel in ("commentary", "analysis"):
525
+ if current_recipient.startswith("functions."):
526
+ rid = random_uuid()
527
+ return [
528
+ ResponseFunctionToolCall(
529
+ arguments=parser.current_content,
530
+ call_id=f"call_{rid}",
531
+ type="function_call",
532
+ name=current_recipient.split(".")[-1],
533
+ id=f"fc_{rid}",
534
+ status="in_progress",
535
)
497
- ],
498
- status=None,
499
- )
500
- return [reasoning_item]
501
- elif parser.current_channel == "final":
536
+ ]
537
+ # Built-in tools (python, browser, container) should be treated as reasoning
538
+ elif not (
539
+ current_recipient.startswith("python")
540
+ or current_recipient.startswith("browser")
541
+ or current_recipient.startswith("container")
542
+ ):
543
+ # All other recipients are MCP calls
544
+ rid = random_uuid()
545
+ server_label, tool_name = _parse_mcp_recipient(current_recipient)
546
+ return [
547
+ McpCall(
548
+ arguments=parser.current_content,
549
+ type="mcp_call",
550
+ name=tool_name,
551
+ server_label=server_label,
552
+ id=f"mcp_{rid}",
553
+ status="in_progress",
554
+ )
555
+ ]
556
+
557
+ if parser.current_channel == "commentary":
558
+ return [
559
+ ResponseReasoningItem(
560
+ id=f"rs_{random_uuid()}",
561
+ summary=[],
562
+ type="reasoning",
563
+ content=[
564
+ ResponseReasoningTextContent(
565
+ text=parser.current_content, type="reasoning_text"
566
+ )
567
+ ],
568
+ status=None,
569
+ )
570
+ ]
571
+
572
+ if parser.current_channel == "analysis":
573
+ return [
574
+ ResponseReasoningItem(
575
+ id=f"rs_{random_uuid()}",
576
+ summary=[],
577
+ type="reasoning",
578
+ content=[
579
+ ResponseReasoningTextContent(
580
+ text=parser.current_content, type="reasoning_text"
581
+ )
582
+ ],
583
+ status=None,
584
+ )
585
+ ]
586
+
587
+ if parser.current_channel == "final":
588
output_text = ResponseOutputText(
589
text=parser.current_content,
590
annotations=[], # TODO
@@ -515,6 +601,7 @@ def parse_remaining_state(parser: StreamableParser) -> list[ResponseOutputItem]:
601
type="message",
602
)
603
return [text_item]
604
+
605
return []
606
607
vllm/entrypoints/openai/protocol.py
+8
@@ -25,6 +25,10 @@ from openai.types.responses import (
25
ResponseContentPartDoneEvent,
26
ResponseFunctionToolCall,
27
ResponseInputItemParam,
28
+ ResponseMcpCallArgumentsDeltaEvent,
29
+ ResponseMcpCallArgumentsDoneEvent,
30
+ ResponseMcpCallCompletedEvent,
31
+ ResponseMcpCallInProgressEvent,
32
ResponseOutputItem,
33
ResponseOutputItemAddedEvent,
34
ResponseOutputItemDoneEvent,
@@ -1790,6 +1794,10 @@ StreamingResponsesResponse: TypeAlias = (
1794
| ResponseCodeInterpreterCallCodeDoneEvent
1795
| ResponseCodeInterpreterCallInterpretingEvent
1796
| ResponseCodeInterpreterCallCompletedEvent
1797
+ | ResponseMcpCallArgumentsDeltaEvent
1798
+ | ResponseMcpCallArgumentsDoneEvent
1799
+ | ResponseMcpCallInProgressEvent
1800
+ | ResponseMcpCallCompletedEvent
1801
)
1802
1803