A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 17:30 UTC
[responsesAPI][6] Fix multi turn MCP tokenization (#30230)
Signed-off-by: Andrew Xia <axia@fb.com> Co-authored-by: Andrew Xia <axia@fb.com>
Andrew Xia committed
Dec 9, 2025 at 18:13 UTC
c3487aca3425f532730c3433cfbd44e880fce2a8
5 files changed
+110
-13
tests/entrypoints/test_responses_utils.py
+45
-7
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import pytest
5
+from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
6
from openai.types.responses.response_function_tool_call_output_item import (
7
ResponseFunctionToolCallOutputItem,
8
)
@@ -14,7 +15,8 @@ from openai.types.responses.response_reasoning_item import (
15
)
16
17
from vllm.entrypoints.responses_utils import (
17
- construct_chat_message_with_tool_call,
18
+ _construct_single_message_from_response_item,
19
+ construct_chat_messages_with_tool_call,
20
convert_tool_responses_to_completions_format,
21
)
22
@@ -42,7 +44,43 @@ class TestResponsesUtils:
44
45
assert result == {"type": "function", "function": input_tool}
46
45
- def test_construct_chat_message_with_tool_call(self):
47
+ def test_construct_chat_messages_with_tool_call(self):
48
+ """Test construction of chat messages with tool calls."""
49
+ reasoning_item = ResponseReasoningItem(
50
+ id="lol",
51
+ summary=[],
52
+ type="reasoning",
53
+ content=[
54
+ Content(
55
+ text="Leroy Jenkins",
56
+ type="reasoning_text",
57
+ )
58
+ ],
59
+ encrypted_content=None,
60
+ status=None,
61
+ )
62
+ mcp_tool_item = ResponseFunctionToolCall(
63
+ id="mcp_123",
64
+ call_id="call_123",
65
+ type="function_call",
66
+ status="completed",
67
+ name="python",
68
+ arguments='{"code": "123+456"}',
69
+ )
70
+ input_items = [reasoning_item, mcp_tool_item]
71
+ messages = construct_chat_messages_with_tool_call(input_items)
72
+
73
+ assert len(messages) == 1
74
+ message = messages[0]
75
+ assert message["role"] == "assistant"
76
+ assert message["reasoning"] == "Leroy Jenkins"
77
+ assert message["tool_calls"][0]["id"] == "call_123"
78
+ assert message["tool_calls"][0]["function"]["name"] == "python"
79
+ assert (
80
+ message["tool_calls"][0]["function"]["arguments"] == '{"code": "123+456"}'
81
+ )
82
+
83
+ def test_construct_single_message_from_response_item(self):
84
item = ResponseReasoningItem(
85
id="lol",
86
summary=[],
@@ -56,7 +94,7 @@ class TestResponsesUtils:
94
encrypted_content=None,
95
status=None,
96
)
59
- formatted_item = construct_chat_message_with_tool_call(item)
97
+ formatted_item = _construct_single_message_from_response_item(item)
98
assert formatted_item["role"] == "assistant"
99
assert formatted_item["reasoning"] == "Leroy Jenkins"
100
@@ -74,7 +112,7 @@ class TestResponsesUtils:
112
status=None,
113
)
114
77
- formatted_item = construct_chat_message_with_tool_call(item)
115
+ formatted_item = _construct_single_message_from_response_item(item)
116
assert formatted_item["role"] == "assistant"
117
assert (
118
formatted_item["reasoning"]
@@ -88,7 +126,7 @@ class TestResponsesUtils:
126
output="1234",
127
status="completed",
128
)
91
- formatted_item = construct_chat_message_with_tool_call(tool_call_output)
129
+ formatted_item = _construct_single_message_from_response_item(tool_call_output)
130
assert formatted_item["role"] == "tool"
131
assert formatted_item["content"] == "1234"
132
assert formatted_item["tool_call_id"] == "temp"
@@ -102,7 +140,7 @@ class TestResponsesUtils:
140
status=None,
141
)
142
with pytest.raises(ValueError):
105
- construct_chat_message_with_tool_call(item)
143
+ _construct_single_message_from_response_item(item)
144
145
output_item = ResponseOutputMessage(
146
id="msg_bf585bbbe3d500e0",
@@ -119,6 +157,6 @@ class TestResponsesUtils:
157
type="message",
158
)
159
122
- formatted_item = construct_chat_message_with_tool_call(output_item)
160
+ formatted_item = _construct_single_message_from_response_item(output_item)
161
assert formatted_item["role"] == "assistant"
162
assert formatted_item["content"] == "dongyi"
vllm/entrypoints/constants.py
+2
@@ -8,3 +8,5 @@ Shared constants for vLLM entrypoints.
8
# These constants help mitigate header abuse attacks
9
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT = 4194304 # 4 MB
10
H11_MAX_HEADER_COUNT_DEFAULT = 256
11
+
12
+MCP_PREFIX = "mcp_"
vllm/entrypoints/context.py
+5
-1
@@ -19,6 +19,7 @@ from vllm import envs
19
from vllm.entrypoints.chat_utils import (
20
ChatTemplateContentFormatOption,
21
)
22
+from vllm.entrypoints.constants import MCP_PREFIX
23
from vllm.entrypoints.openai.parser.harmony_utils import (
24
get_encoding,
25
get_streamable_parser_for_assistant,
@@ -303,7 +304,7 @@ class ParsableContext(ConversationContext):
304
result_str = result.content[0].text
305
306
message = ResponseFunctionToolCallOutputItem(
306
- id=f"fco_{random_uuid()}",
307
+ id=f"mcpo_{random_uuid()}",
308
type="function_call_output",
309
call_id=f"call_{random_uuid()}",
310
output=result_str,
@@ -385,6 +386,9 @@ class ParsableContext(ConversationContext):
386
if not self.parser.response_messages:
387
return []
388
last_msg = self.parser.response_messages[-1]
389
+ # change this to a mcp_ function call
390
+ last_msg.id = f"{MCP_PREFIX}{random_uuid()}"
391
+ self.parser.response_messages[-1] = last_msg
392
if last_msg.name == "code_interpreter":
393
return await self.call_python_tool(self._tool_sessions["python"], last_msg)
394
elif last_msg.name == "web_search_preview":
vllm/entrypoints/openai/serving_engine.py
+1
@@ -1339,6 +1339,7 @@ class OpenAIServing:
1339
)
1340
engine_prompt = engine_prompts[0]
1341
request_prompt = request_prompts[0]
1342
+ prompt_text, _, _ = self._get_prompt_components(request_prompt)
1343
1344
# Update the sampling params.
1345
sampling_params.max_tokens = self.max_model_len - len(
vllm/entrypoints/responses_utils.py
+57
-5
@@ -22,6 +22,7 @@ from openai.types.responses.response_reasoning_item import ResponseReasoningItem
22
from openai.types.responses.tool import Tool
23
24
from vllm import envs
25
+from vllm.entrypoints.constants import MCP_PREFIX
26
from vllm.entrypoints.openai.protocol import (
27
ChatCompletionMessageParam,
28
ResponseInputOutputItem,
@@ -44,13 +45,13 @@ def make_response_output_items_from_parsable_context(
45
)
46
if isinstance(output_messages[-1], ResponseFunctionToolCall):
47
mcp_message = McpCall(
47
- id=f"mcp_{random_uuid()}",
48
+ id=f"{MCP_PREFIX}{random_uuid()}",
49
arguments=output_messages[-1].arguments,
50
name=output_messages[-1].name,
51
server_label=output_messages[
52
-1
53
].name, # TODO: store the server label
53
- type="mcp_call",
54
+ type=f"{MCP_PREFIX}call",
55
status="completed",
56
output=message.output,
57
# TODO: support error output
@@ -98,12 +99,63 @@ def construct_input_messages(
99
if isinstance(request_input, str):
100
messages.append({"role": "user", "content": request_input})
101
else:
101
- for item in request_input:
102
- messages.append(construct_chat_message_with_tool_call(item))
102
+ input_messages = construct_chat_messages_with_tool_call(request_input)
103
+ messages.extend(input_messages)
104
return messages
105
106
106
-def construct_chat_message_with_tool_call(
107
+def _maybe_combine_reasoning_and_tool_call(
108
+ item: ResponseInputOutputItem, messages: list[ChatCompletionMessageParam]
109
+) -> ChatCompletionMessageParam | None:
110
+ """Many models treat MCP calls and reasoning as a single message.
111
+ This function checks if the last message is a reasoning message and
112
+ the current message is a tool call"""
113
+ if not (
114
+ isinstance(item, ResponseFunctionToolCall) and item.id.startswith(MCP_PREFIX)
115
+ ):
116
+ return None
117
+ if len(messages) == 0:
118
+ return None
119
+ last_message = messages[-1]
120
+ if not (
121
+ last_message.get("role") == "assistant"
122
+ and last_message.get("reasoning") is not None
123
+ ):
124
+ return None
125
+
126
+ last_message["tool_calls"] = [
127
+ ChatCompletionMessageToolCallParam(
128
+ id=item.call_id,
129
+ function=FunctionCallTool(
130
+ name=item.name,
131
+ arguments=item.arguments,
132
+ ),
133
+ type="function",
134
+ )
135
+ ]
136
+ return last_message
137
+
138
+
139
+def construct_chat_messages_with_tool_call(
140
+ input_messages: list[ResponseInputOutputItem],
141
+) -> list[ChatCompletionMessageParam]:
142
+ """This function wraps _construct_single_message_from_response_item
143
+ Because some chatMessages come from multiple response items
144
+ for example a reasoning item and a MCP tool call are two response items
145
+ but are one chat message
146
+ """
147
+ messages: list[ChatCompletionMessageParam] = []
148
+ for item in input_messages:
149
+ maybe_combined_message = _maybe_combine_reasoning_and_tool_call(item, messages)
150
+ if maybe_combined_message is not None:
151
+ messages[-1] = maybe_combined_message
152
+ else:
153
+ messages.append(_construct_single_message_from_response_item(item))
154
+
155
+ return messages
156
+
157
+
158
+def _construct_single_message_from_response_item(
159
item: ResponseInputOutputItem,
160
) -> ChatCompletionMessageParam:
161
if isinstance(item, ResponseFunctionToolCall):