A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 20:15 UTC
[Bugfix] [Frontend] Responses API, fix merging of messages (#42189)
Signed-off-by: Yifan Zong <yzong@redhat.com> Signed-off-by: Yifan <yzong@redhat.com>
yzong-rh committed
May 12, 2026 at 12:09 UTC
6ff7405b81317932cf2e97c7e18dcbae4db7542c
2 files changed
+298
-189
tests/entrypoints/openai/responses/test_responses_utils.py
+234
-130
@@ -4,7 +4,6 @@
4
from unittest.mock import patch
5
6
import pytest
7
-from openai.types.chat import ChatCompletionMessageParam
7
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
8
from openai.types.responses.response_function_tool_call_output_item import (
9
ResponseFunctionToolCallOutputItem,
@@ -17,10 +16,8 @@ from openai.types.responses.response_reasoning_item import (
16
Summary,
17
)
18
20
-from vllm.entrypoints.constants import MCP_PREFIX
19
from vllm.entrypoints.openai.responses.utils import (
22
- _construct_single_message_from_response_item,
23
- _maybe_combine_reasoning_and_tool_call,
20
+ _construct_message_from_response_item,
21
construct_chat_messages_with_tool_call,
22
construct_input_messages,
23
convert_tool_responses_to_completions_format,
@@ -28,6 +25,96 @@ from vllm.entrypoints.openai.responses.utils import (
25
)
26
27
28
+def _single_chat_message(item):
29
+ message = _construct_message_from_response_item(item)
30
+ assert message is not None
31
+ return message
32
+
33
+
34
+def make_output_message(
35
+ text: str,
36
+ *,
37
+ id: str = "msg_1",
38
+ status: str = "completed",
39
+) -> ResponseOutputMessage:
40
+ return ResponseOutputMessage(
41
+ id=id,
42
+ content=[
43
+ ResponseOutputText(
44
+ annotations=[],
45
+ text=text,
46
+ type="output_text",
47
+ logprobs=None,
48
+ )
49
+ ],
50
+ role="assistant",
51
+ status=status,
52
+ type="message",
53
+ )
54
+
55
+
56
+def make_reasoning_item(
57
+ *,
58
+ content_text: str | None = None,
59
+ summary_text: str | None = None,
60
+ content: list[Content] | None = None,
61
+ summary: list[Summary] | None = None,
62
+ encrypted_content: str | None = None,
63
+ id: str = "reasoning_1",
64
+ status: str | None = None,
65
+) -> ResponseReasoningItem:
66
+ if content is None and content_text is not None:
67
+ content = [Content(text=content_text, type="reasoning_text")]
68
+ if summary is None and summary_text is not None:
69
+ summary = [Summary(text=summary_text, type="summary_text")]
70
+
71
+ return ResponseReasoningItem(
72
+ id=id,
73
+ summary=[] if summary is None else summary,
74
+ type="reasoning",
75
+ content=content,
76
+ encrypted_content=encrypted_content,
77
+ status=status,
78
+ )
79
+
80
+
81
+def make_function_call(
82
+ *,
83
+ call_id: str,
84
+ name: str = "test_function",
85
+ arguments: str = "{}",
86
+ id: str = "tool_id",
87
+ status: str | None = None,
88
+) -> ResponseFunctionToolCall:
89
+ kwargs = {
90
+ "type": "function_call",
91
+ "id": id,
92
+ "call_id": call_id,
93
+ "name": name,
94
+ "arguments": arguments,
95
+ }
96
+ if status is not None:
97
+ kwargs["status"] = status
98
+
99
+ return ResponseFunctionToolCall(**kwargs)
100
+
101
+
102
+def make_function_call_output(
103
+ *,
104
+ call_id: str,
105
+ output: str = "42",
106
+ id: str = "output_1",
107
+ status: str = "completed",
108
+) -> ResponseFunctionToolCallOutputItem:
109
+ return ResponseFunctionToolCallOutputItem(
110
+ id=id,
111
+ type="function_call_output",
112
+ call_id=call_id,
113
+ output=output,
114
+ status=status,
115
+ )
116
+
117
+
118
class TestResponsesUtils:
119
"""Tests for convert_tool_responses_to_completions_format function."""
120
@@ -87,7 +174,7 @@ class TestResponsesUtils:
174
message["tool_calls"][0]["function"]["arguments"] == '{"code": "123+456"}'
175
)
176
90
- def test_construct_single_message_from_response_item(self):
177
+ def test_construct_chat_messages_preserves_single_item_conversions(self):
178
item = ResponseReasoningItem(
179
id="lol",
180
summary=[],
@@ -101,7 +188,7 @@ class TestResponsesUtils:
188
encrypted_content=None,
189
status=None,
190
)
104
- formatted_item = _construct_single_message_from_response_item(item)
191
+ formatted_item = _single_chat_message(item)
192
assert formatted_item["role"] == "assistant"
193
assert formatted_item["reasoning"] == "Leroy Jenkins"
194
@@ -119,7 +206,7 @@ class TestResponsesUtils:
206
status=None,
207
)
208
122
- formatted_item = _construct_single_message_from_response_item(item)
209
+ formatted_item = _single_chat_message(item)
210
assert formatted_item["role"] == "assistant"
211
assert (
212
formatted_item["reasoning"]
@@ -133,11 +220,22 @@ class TestResponsesUtils:
220
output="1234",
221
status="completed",
222
)
136
- formatted_item = _construct_single_message_from_response_item(tool_call_output)
223
+ formatted_item = _single_chat_message(tool_call_output)
224
assert formatted_item["role"] == "tool"
225
assert formatted_item["content"] == "1234"
226
assert formatted_item["tool_call_id"] == "temp"
227
228
+ formatted_item = _single_chat_message(
229
+ {
230
+ "type": "function_call_output",
231
+ "call_id": "temp_dict",
232
+ "output": "5678",
233
+ }
234
+ )
235
+ assert formatted_item["role"] == "tool"
236
+ assert formatted_item["content"] == "5678"
237
+ assert formatted_item["tool_call_id"] == "temp_dict"
238
+
239
item = ResponseReasoningItem(
240
id="lol",
241
summary=[],
@@ -147,7 +245,7 @@ class TestResponsesUtils:
245
status=None,
246
)
247
with pytest.raises(ValueError):
150
- _construct_single_message_from_response_item(item)
248
+ construct_chat_messages_with_tool_call([item])
249
250
output_item = ResponseOutputMessage(
251
id="msg_bf585bbbe3d500e0",
@@ -164,7 +262,7 @@ class TestResponsesUtils:
262
type="message",
263
)
264
167
- formatted_item = _construct_single_message_from_response_item(output_item)
265
+ formatted_item = _single_chat_message(output_item)
266
assert formatted_item["role"] == "assistant"
267
assert formatted_item["content"] == "dongyi"
268
@@ -192,7 +290,7 @@ class TestReasoningItemContentPriority:
290
encrypted_content=None,
291
status=None,
292
)
195
- formatted = _construct_single_message_from_response_item(item)
293
+ formatted = _single_chat_message(item)
294
assert formatted["reasoning"] == "This is the actual content"
295
296
def test_content_only(self):
@@ -210,7 +308,7 @@ class TestReasoningItemContentPriority:
308
encrypted_content=None,
309
status=None,
310
)
213
- formatted = _construct_single_message_from_response_item(item)
311
+ formatted = _single_chat_message(item)
312
assert formatted["reasoning"] == "Content without summary"
313
314
@patch("vllm.entrypoints.openai.responses.utils.logger")
@@ -229,7 +327,7 @@ class TestReasoningItemContentPriority:
327
encrypted_content=None,
328
status=None,
329
)
232
- formatted = _construct_single_message_from_response_item(item)
330
+ formatted = _single_chat_message(item)
331
assert formatted["reasoning"] == "Fallback summary text"
332
mock_logger.warning.assert_called_once()
333
assert (
@@ -252,7 +350,7 @@ class TestReasoningItemContentPriority:
350
encrypted_content=None,
351
status=None,
352
)
255
- formatted = _construct_single_message_from_response_item(item)
353
+ formatted = _single_chat_message(item)
354
assert formatted["reasoning"] == "Summary when content empty"
355
mock_logger.warning.assert_called_once()
356
assert (
@@ -269,7 +367,7 @@ class TestReasoningItemContentPriority:
367
encrypted_content=None,
368
status=None,
369
)
272
- formatted = _construct_single_message_from_response_item(item)
370
+ formatted = _single_chat_message(item)
371
assert formatted["reasoning"] == ""
372
373
def test_encrypted_content_raises(self):
@@ -293,7 +391,7 @@ class TestReasoningItemContentPriority:
391
status=None,
392
)
393
with pytest.raises(ValueError):
296
- _construct_single_message_from_response_item(item)
394
+ construct_chat_messages_with_tool_call([item])
395
396
@patch("vllm.entrypoints.openai.responses.utils.logger")
397
def test_summary_with_multiple_entries_uses_first(self, mock_logger):
@@ -315,7 +413,7 @@ class TestReasoningItemContentPriority:
413
encrypted_content=None,
414
status=None,
415
)
318
- formatted = _construct_single_message_from_response_item(item)
416
+ formatted = _single_chat_message(item)
417
assert formatted["reasoning"] == "First summary"
418
mock_logger.warning.assert_called_once()
419
assert (
@@ -343,7 +441,7 @@ class TestReasoningItemContentPriority:
441
encrypted_content=None,
442
status=None,
443
)
346
- _construct_single_message_from_response_item(item)
444
+ construct_chat_messages_with_tool_call([item])
445
mock_logger.warning.assert_not_called()
446
447
@@ -626,119 +724,125 @@ class TestShouldContinueFinalMessage:
724
assert should_continue_final_message([dict_item]) is False
725
726
629
-class TestMaybeCombineReasoningAndToolCall:
630
- """Tests for _maybe_combine_reasoning_and_tool_call function."""
631
-
632
- def test_returns_none_when_item_id_is_none(self):
633
- """
634
- Test fix from PR #31999: when item.id is None, should return None
635
- instead of raising TypeError on startswith().
636
- """
637
- item = ResponseFunctionToolCall(
638
- type="function_call",
639
- id=None, # This was causing TypeError before the fix
640
- call_id="call_123",
641
- name="test_function",
642
- arguments="{}",
643
- )
644
- messages: list[ChatCompletionMessageParam] = []
645
-
646
- result = _maybe_combine_reasoning_and_tool_call(item, messages)
647
-
648
- assert result is None
649
-
650
- def test_returns_none_when_id_does_not_start_with_mcp_prefix(self):
651
- """Test that non-MCP tool calls are not combined."""
652
- item = ResponseFunctionToolCall(
653
- type="function_call",
654
- id="regular_id", # Does not start with MCP_PREFIX
655
- call_id="call_123",
656
- name="test_function",
657
- arguments="{}",
658
- )
659
- messages = [{"role": "assistant", "reasoning": "some reasoning"}]
660
-
661
- result = _maybe_combine_reasoning_and_tool_call(item, messages)
662
-
663
- assert result is None
664
-
665
- def test_returns_none_when_last_message_is_not_assistant(self):
666
- """Test that non-assistant last message returns None."""
667
- item = ResponseFunctionToolCall(
668
- type="function_call",
669
- id=f"{MCP_PREFIX}tool_id",
670
- call_id="call_123",
671
- name="test_function",
672
- arguments="{}",
673
- )
674
- messages = [{"role": "user", "content": "hello"}]
675
-
676
- result = _maybe_combine_reasoning_and_tool_call(item, messages)
677
-
678
- assert result is None
679
-
680
- def test_returns_none_when_last_message_has_no_reasoning(self):
681
- """Test that assistant message without reasoning returns None."""
682
- item = ResponseFunctionToolCall(
683
- type="function_call",
684
- id=f"{MCP_PREFIX}tool_id",
685
- call_id="call_123",
686
- name="test_function",
687
- arguments="{}",
688
- )
689
- messages = [{"role": "assistant", "content": "some content"}]
690
-
691
- result = _maybe_combine_reasoning_and_tool_call(item, messages)
692
-
693
- assert result is None
694
-
695
- def test_combines_reasoning_and_mcp_tool_call(self):
696
- """Test successful combination of reasoning message and MCP tool call."""
697
- item = ResponseFunctionToolCall(
698
- type="function_call",
699
- id=f"{MCP_PREFIX}tool_id",
700
- call_id="call_123",
701
- name="test_function",
702
- arguments='{"arg": "value"}',
703
- )
704
- messages = [{"role": "assistant", "reasoning": "I need to call this tool"}]
727
+class TestConstructChatMessagesCombinePolicy:
728
+ """Tests for contiguous assistant-side merging."""
729
+
730
+ @pytest.mark.parametrize(
731
+ ("items", "expected_content", "expected_reasoning", "expected_tool_call_ids"),
732
+ [
733
+ pytest.param(
734
+ [
735
+ make_reasoning_item(content_text="Let me think"),
736
+ make_output_message("Hello"),
737
+ ],
738
+ "Hello",
739
+ "Let me think",
740
+ None,
741
+ id="reasoning-output-messages",
742
+ ),
743
+ pytest.param(
744
+ [
745
+ make_function_call(call_id="call_123"),
746
+ make_function_call(call_id="call_456"),
747
+ ],
748
+ None,
749
+ None,
750
+ ["call_123", "call_456"],
751
+ id="consecutive-tool-calls",
752
+ ),
753
+ pytest.param(
754
+ [
755
+ make_reasoning_item(content_text="Let me think"),
756
+ make_function_call(call_id="call_123"),
757
+ ],
758
+ None,
759
+ "Let me think",
760
+ ["call_123"],
761
+ id="reasoning-tool-call",
762
+ ),
763
+ pytest.param(
764
+ [
765
+ make_output_message("Hello"),
766
+ make_function_call(call_id="call_123"),
767
+ ],
768
+ "Hello",
769
+ None,
770
+ ["call_123"],
771
+ id="output-tool-call",
772
+ ),
773
+ pytest.param(
774
+ [
775
+ make_reasoning_item(content_text="Thinking"),
776
+ make_output_message("Hello"),
777
+ make_function_call(call_id="call_123"),
778
+ make_function_call(call_id="call_456"),
779
+ ],
780
+ "Hello",
781
+ "Thinking",
782
+ ["call_123", "call_456"],
783
+ id="reasoning-output-tool-call",
784
+ ),
785
+ ],
786
+ )
787
+ def test_assistant_side_items_merge_until_tool_output(
788
+ self,
789
+ items,
790
+ expected_content,
791
+ expected_reasoning,
792
+ expected_tool_call_ids,
793
+ ):
794
+ messages = construct_chat_messages_with_tool_call(items)
795
706
- result = _maybe_combine_reasoning_and_tool_call(item, messages)
707
-
708
- assert result is not None
709
- assert result["role"] == "assistant"
710
- assert result["reasoning"] == "I need to call this tool"
711
- assert "tool_calls" in result
712
- assert len(result["tool_calls"]) == 1
713
- assert result["tool_calls"][0]["id"] == "call_123"
714
- assert result["tool_calls"][0]["function"]["name"] == "test_function"
715
- assert result["tool_calls"][0]["function"]["arguments"] == '{"arg": "value"}'
716
- assert result["tool_calls"][0]["type"] == "function"
717
-
718
- def test_returns_none_for_non_function_tool_call_type(self):
719
- """Test that non-ResponseFunctionToolCall items return None."""
720
- # Pass a dict instead of ResponseFunctionToolCall
721
- item = {"type": "message", "content": "hello"}
722
- messages = [{"role": "assistant", "reasoning": "some reasoning"}]
723
-
724
- result = _maybe_combine_reasoning_and_tool_call(item, messages)
725
-
726
- assert result is None
727
-
728
- def test_returns_none_when_id_is_empty_string(self):
729
- """Test that empty string id returns None (falsy check)."""
730
- item = ResponseFunctionToolCall(
731
- type="function_call",
732
- id="", # Empty string is falsy
733
- call_id="call_123",
734
- name="test_function",
735
- arguments="{}",
736
- )
737
- messages = [{"role": "assistant", "reasoning": "some reasoning"}]
738
-
739
- result = _maybe_combine_reasoning_and_tool_call(item, messages)
740
-
741
- assert result is None
796
+ assert len(messages) == 1
797
+ assert messages[0]["role"] == "assistant"
798
+ if expected_content is None:
799
+ assert "content" not in messages[0]
800
+ else:
801
+ assert messages[0]["content"] == expected_content
802
+ if expected_reasoning is None:
803
+ assert "reasoning" not in messages[0]
804
+ else:
805
+ assert messages[0]["reasoning"] == expected_reasoning
806
+ if expected_tool_call_ids is None:
807
+ assert "tool_calls" not in messages[0]
808
+ else:
809
+ assert [tool_call["id"] for tool_call in messages[0]["tool_calls"]] == (
810
+ expected_tool_call_ids
811
+ )
812
+
813
+ @pytest.mark.parametrize(
814
+ ("items", "num_expected_messages"),
815
+ [
816
+ pytest.param(
817
+ [
818
+ make_output_message("Hello"),
819
+ make_output_message("World"),
820
+ ],
821
+ 2,
822
+ id="consecutive-output-messages",
823
+ ),
824
+ pytest.param(
825
+ [
826
+ make_reasoning_item(content_text="Let me think"),
827
+ make_reasoning_item(content_text="Let me think more"),
828
+ ],
829
+ 2,
830
+ id="consecutive-reasoning-messages",
831
+ ),
832
+ pytest.param(
833
+ [
834
+ make_function_call(call_id="call_123"),
835
+ make_function_call_output(call_id="call_123", output="42"),
836
+ make_function_call(call_id="call_456"),
837
+ ],
838
+ 3,
839
+ id="interrupted-by-non-assistant-item",
840
+ ),
841
+ ],
842
+ )
843
+ def test_merge_chain_breaks(self, items, num_expected_messages):
844
+ messages = construct_chat_messages_with_tool_call(items)
845
+ assert len(messages) == num_expected_messages
846
847
848
class TestConstructInputMessagesInstructionsLeak:
vllm/entrypoints/openai/responses/utils.py
+64
-59
@@ -1,6 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from collections.abc import Iterable
5
from typing import Any
6
7
from openai.types.chat import (
@@ -21,7 +22,6 @@ from openai.types.responses.response_reasoning_item import ResponseReasoningItem
22
from openai.types.responses.tool import Tool
23
24
from vllm import envs
24
-from vllm.entrypoints.constants import MCP_PREFIX
25
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionMessageParam
26
from vllm.entrypoints.openai.responses.protocol import ResponseInputOutputItem
27
from vllm.logger import init_logger
@@ -121,76 +121,69 @@ def construct_input_messages(
121
return messages
122
123
124
-def _maybe_combine_reasoning_and_tool_call(
125
- item: ResponseInputOutputItem, messages: list[ChatCompletionMessageParam]
126
-) -> ChatCompletionMessageParam | None:
127
- """Many models treat MCP calls and reasoning as a single message.
128
- This function checks if the last message is a reasoning message and
129
- the current message is a tool call"""
130
- if not (
131
- isinstance(item, ResponseFunctionToolCall)
132
- and item.id
133
- and item.id.startswith(MCP_PREFIX)
134
- ):
135
- return None
136
- if len(messages) == 0:
137
- return None
138
- last_message = messages[-1]
139
- if not (
140
- last_message.get("role") == "assistant"
141
- and last_message.get("reasoning") is not None
142
- ):
143
- return None
144
-
145
- last_message["tool_calls"] = [
146
- ChatCompletionMessageToolCallParam(
147
- id=item.call_id,
148
- function=FunctionCallTool(
149
- name=item.name,
150
- arguments=item.arguments,
151
- ),
152
- type="function",
153
- )
154
- ]
155
- return last_message
156
-
157
-
124
def construct_chat_messages_with_tool_call(
125
input_messages: list[ResponseInputOutputItem],
126
) -> list[ChatCompletionMessageParam]:
161
- """This function wraps _construct_single_message_from_response_item
162
- Because some chatMessages come from multiple response items
163
- for example a reasoning item and a MCP tool call are two response items
164
- but are one chat message
127
+ """Build chat messages from response items.
128
+
129
+ Some chat messages span multiple response items (e.g., reasoning + tool calls).
130
"""
131
messages: list[ChatCompletionMessageParam] = []
132
for item in input_messages:
168
- maybe_combined_message = _maybe_combine_reasoning_and_tool_call(item, messages)
169
- if maybe_combined_message is not None:
170
- messages[-1] = maybe_combined_message
171
- else:
172
- messages.append(_construct_single_message_from_response_item(item))
133
+ message = _construct_message_from_response_item(
134
+ item, prev_msg=messages[-1] if messages else None
135
+ )
136
+ if message is not None:
137
+ messages.append(message)
138
139
return messages
140
141
177
-def _construct_single_message_from_response_item(
142
+def _construct_message_from_response_item(
143
item: ResponseInputOutputItem,
179
-) -> ChatCompletionMessageParam:
144
+ prev_msg: ChatCompletionMessageParam | None = None,
145
+) -> ChatCompletionMessageParam | None:
146
+ """
147
+ Returns a new message or None. If `None`, `prev_msg` might be updated.
148
+ If `prev_msg` is `None`, a new message is always returned.
149
+ """
150
+ prev_assistant_msg = (
151
+ prev_msg if prev_msg and prev_msg.get("role") == "assistant" else None
152
+ )
153
+
154
if isinstance(item, ResponseFunctionToolCall):
181
- # Append the function call as a tool call.
155
+ tool_call = ChatCompletionMessageToolCallParam(
156
+ id=item.call_id,
157
+ function=FunctionCallTool(
158
+ name=item.name,
159
+ arguments=item.arguments,
160
+ ),
161
+ type="function",
162
+ )
163
+ if prev_assistant_msg:
164
+ tool_calls = prev_assistant_msg.get("tool_calls")
165
+ if tool_calls is None:
166
+ prev_assistant_msg["tool_calls"] = [tool_call]
167
+ return None
168
+ if isinstance(tool_calls, list):
169
+ tool_calls.append(tool_call)
170
+ return None
171
+ if isinstance(tool_calls, Iterable) and not isinstance(
172
+ tool_calls, (dict, str)
173
+ ):
174
+ tool_calls = list(tool_calls)
175
+ tool_calls.append(tool_call)
176
+ prev_assistant_msg["tool_calls"] = tool_calls
177
+ return None
178
+ logger.warning(
179
+ "Previous assistant message has unknown tool_calls format. "
180
+ "Tool call merging is skipped and a new assistant message is created. "
181
+ "Item %s",
182
+ item.id,
183
+ )
184
return ChatCompletionAssistantMessageParam(
185
role="assistant",
184
- tool_calls=[
185
- ChatCompletionMessageToolCallParam(
186
- id=item.call_id,
187
- function=FunctionCallTool(
188
- name=item.name,
189
- arguments=item.arguments,
190
- ),
191
- type="function",
192
- )
193
- ],
186
+ tool_calls=[tool_call],
187
)
188
elif isinstance(item, ResponseReasoningItem):
189
reasoning = ""
@@ -206,14 +199,26 @@ def _construct_single_message_from_response_item(
199
"reasoning items.",
200
item.id,
201
)
202
+
203
+ if prev_assistant_msg:
204
+ previous_reasoning = prev_assistant_msg.get("reasoning")
205
+ if previous_reasoning is None:
206
+ prev_assistant_msg["reasoning"] = reasoning
207
+ return None
208
return {
209
"role": "assistant",
210
"reasoning": reasoning,
211
}
212
elif isinstance(item, ResponseOutputMessage):
213
+ output_text = item.content[0].text
214
+ if prev_assistant_msg:
215
+ previous_content = prev_assistant_msg.get("content")
216
+ if previous_content is None:
217
+ prev_assistant_msg["content"] = output_text
218
+ return None
219
return {
220
"role": "assistant",
216
- "content": item.content[0].text,
221
+ "content": output_text,
222
}
223
elif isinstance(item, ResponseFunctionToolCallOutputItem):
224
return ChatCompletionToolMessageParam(
@@ -228,7 +233,7 @@ def _construct_single_message_from_response_item(
233
content=item.get("output"),
234
tool_call_id=item.get("call_id"),
235
)
231
- return item # type: ignore
236
+ return item # type: ignore[arg-type]
237
238
239
def extract_tool_types(tools: list[Tool]) -> set[str]: