@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 53a56e658b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[gpt-oss][2/N] Support input_messages in responsesRequest (#26962)

Signed-off-by: Andrew Xia <axia@fb.com> Co-authored-by: Andrew Xia <axia@fb.com>

Andrew Xia committed Oct 27, 2025 at 16:15 UTC 53a56e658b9ab1eabb7339d3001dc3fd9178dd21
7 files changed +481 -17
tests/entrypoints/openai/test_response_api_with_harmony.py
+129
@@ -535,11 +535,17 @@ def get_place_to_travel():
535 return "Paris"
536
537
538 +def get_horoscope(sign):
539 + return f"{sign}: Next Tuesday you will befriend a baby otter."
540 +
541 +
542 def call_function(name, args):
543 if name == "get_weather":
544 return get_weather(**args)
545 elif name == "get_place_to_travel":
546 return get_place_to_travel()
547 + elif name == "get_horoscope":
548 + return get_horoscope(**args)
549 else:
550 raise ValueError(f"Unknown function: {name}")
551
@@ -828,3 +834,126 @@ async def test_output_messages_enabled(client: OpenAI, model_name: str, server):
834 assert response.status == "completed"
835 assert len(response.input_messages) > 0
836 assert len(response.output_messages) > 0
837 +
838 +
839 +@pytest.mark.asyncio
840 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
841 +async def test_function_call_with_previous_input_messages(
842 + client: OpenAI, model_name: str
843 +):
844 + """Test function calling using previous_input_messages
845 + for multi-turn conversation with a function call"""
846 +
847 + # Define the get_horoscope tool
848 + tools = [
849 + {
850 + "type": "function",
851 + "name": "get_horoscope",
852 + "description": "Get today's horoscope for an astrological sign.",
853 + "parameters": {
854 + "type": "object",
855 + "properties": {
856 + "sign": {"type": "string"},
857 + },
858 + "required": ["sign"],
859 + "additionalProperties": False,
860 + },
861 + "strict": True,
862 + }
863 + ]
864 +
865 + # Step 1: First call with the function tool
866 + stream_response = await client.responses.create(
867 + model=model_name,
868 + input="What is the horoscope for Aquarius today?",
869 + tools=tools,
870 + extra_body={"enable_response_messages": True},
871 + stream=True,
872 + )
873 +
874 + response = None
875 + async for event in stream_response:
876 + if event.type == "response.completed":
877 + response = event.response
878 +
879 + assert response is not None
880 + assert response.status == "completed"
881 +
882 + # Step 2: Parse the first output to find the function_call type
883 + function_call = None
884 + for item in response.output:
885 + if item.type == "function_call":
886 + function_call = item
887 + break
888 +
889 + assert function_call is not None, "Expected a function_call in the output"
890 + assert function_call.name == "get_horoscope"
891 + assert function_call.call_id is not None
892 +
893 + # Verify the format matches expectations
894 + args = json.loads(function_call.arguments)
895 + assert "sign" in args
896 +
897 + # Step 3: Call the get_horoscope function
898 + result = call_function(function_call.name, args)
899 + assert "Aquarius" in result
900 + assert "baby otter" in result
901 +
902 + # Get the input_messages and output_messages from the first response
903 + first_input_messages = response.input_messages
904 + first_output_messages = response.output_messages
905 +
906 + # Construct the full conversation history using previous_input_messages
907 + previous_messages = (
908 + first_input_messages
909 + + first_output_messages
910 + + [
911 + {
912 + "role": "tool",
913 + "name": "functions.get_horoscope",
914 + "content": [{"type": "text", "text": str(result)}],
915 + }
916 + ]
917 + )
918 +
919 + # Step 4: Make another responses.create() call with previous_input_messages
920 + stream_response_2 = await client.responses.create(
921 + model=model_name,
922 + tools=tools,
923 + input="",
924 + extra_body={
925 + "previous_input_messages": previous_messages,
926 + "enable_response_messages": True,
927 + },
928 + stream=True,
929 + )
930 +
931 + async for event in stream_response_2:
932 + if event.type == "response.completed":
933 + response_2 = event.response
934 +
935 + assert response_2 is not None
936 + assert response_2.status == "completed"
937 + assert response_2.output_text is not None
938 +
939 + # verify only one system message / developer message
940 + num_system_messages_input = 0
941 + num_developer_messages_input = 0
942 + num_function_call_input = 0
943 + for message_dict in response_2.input_messages:
944 + message = Message.from_dict(message_dict)
945 + if message.author.role == "system":
946 + num_system_messages_input += 1
947 + elif message.author.role == "developer":
948 + num_developer_messages_input += 1
949 + elif message.author.role == "tool":
950 + num_function_call_input += 1
951 + assert num_system_messages_input == 1
952 + assert num_developer_messages_input == 1
953 + assert num_function_call_input == 1
954 +
955 + # Verify the output makes sense - should contain information about the horoscope
956 + output_text = response_2.output_text.lower()
957 + assert (
958 + "aquarius" in output_text or "otter" in output_text or "tuesday" in output_text
959 + )
tests/entrypoints/openai/test_serving_responses.py
+22
@@ -125,6 +125,28 @@ class TestInitializeToolSessions:
125 # Verify that init_tool_sessions was called
126 assert mock_context.init_tool_sessions_called
127
128 + def test_validate_create_responses_input(
129 + self, serving_responses_instance, mock_context, mock_exit_stack
130 + ):
131 + request = ResponsesRequest(
132 + input="test input",
133 + previous_input_messages=[
134 + {
135 + "role": "user",
136 + "content": [
137 + {
138 + "type": "text",
139 + "text": "What is my horoscope? I am an Aquarius.",
140 + }
141 + ],
142 + }
143 + ],
144 + previous_response_id="lol",
145 + )
146 + error = serving_responses_instance._validate_create_responses_input(request)
147 + assert error is not None
148 + assert error.error.type == "invalid_request_error"
149 +
150
151 class TestValidateGeneratorInput:
152 """Test class for _validate_generator_input method"""
tests/entrypoints/test_harmony_utils.py new
+254
@@ -0,0 +1,254 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from openai_harmony import Role
5 +
6 +from vllm.entrypoints.harmony_utils import parse_input_to_harmony_message
7 +
8 +
9 +class TestParseInputToHarmonyMessage:
10 + """Tests for parse_input_to_harmony_message function."""
11 +
12 + def test_assistant_message_with_tool_calls(self):
13 + """Test parsing assistant message with tool calls."""
14 + chat_msg = {
15 + "role": "assistant",
16 + "tool_calls": [
17 + {
18 + "function": {
19 + "name": "get_weather",
20 + "arguments": '{"location": "San Francisco"}',
21 + }
22 + },
23 + {
24 + "function": {
25 + "name": "search_web",
26 + "arguments": '{"query": "latest news"}',
27 + }
28 + },
29 + ],
30 + }
31 +
32 + messages = parse_input_to_harmony_message(chat_msg)
33 +
34 + assert len(messages) == 2
35 +
36 + # First tool call
37 + assert messages[0].author.role == Role.ASSISTANT
38 + assert messages[0].content[0].text == '{"location": "San Francisco"}'
39 + assert messages[0].channel == "commentary"
40 + assert messages[0].recipient == "functions.get_weather"
41 + assert messages[0].content_type == "json"
42 +
43 + # Second tool call
44 + assert messages[1].author.role == Role.ASSISTANT
45 + assert messages[1].content[0].text == '{"query": "latest news"}'
46 + assert messages[1].channel == "commentary"
47 + assert messages[1].recipient == "functions.search_web"
48 + assert messages[1].content_type == "json"
49 +
50 + def test_assistant_message_with_empty_tool_call_arguments(self):
51 + """Test parsing assistant message with tool call having None arguments."""
52 + chat_msg = {
53 + "role": "assistant",
54 + "tool_calls": [
55 + {
56 + "function": {
57 + "name": "get_current_time",
58 + "arguments": None,
59 + }
60 + }
61 + ],
62 + }
63 +
64 + messages = parse_input_to_harmony_message(chat_msg)
65 +
66 + assert len(messages) == 1
67 + assert messages[0].content[0].text == ""
68 + assert messages[0].recipient == "functions.get_current_time"
69 +
70 + def test_tool_message_with_string_content(self):
71 + """Test parsing tool message with string content."""
72 + chat_msg = {
73 + "role": "tool",
74 + "name": "get_weather",
75 + "content": "The weather in San Francisco is sunny, 72°F",
76 + }
77 +
78 + messages = parse_input_to_harmony_message(chat_msg)
79 +
80 + assert len(messages) == 1
81 + assert messages[0].author.role == Role.TOOL
82 + assert messages[0].author.name == "functions.get_weather"
83 + assert (
84 + messages[0].content[0].text == "The weather in San Francisco is sunny, 72°F"
85 + )
86 + assert messages[0].channel == "commentary"
87 +
88 + def test_tool_message_with_array_content(self):
89 + """Test parsing tool message with array content."""
90 + chat_msg = {
91 + "role": "tool",
92 + "name": "search_results",
93 + "content": [
94 + {"type": "text", "text": "Result 1: "},
95 + {"type": "text", "text": "Result 2: "},
96 + {
97 + "type": "image",
98 + "url": "http://example.com/img.png",
99 + }, # Should be ignored
100 + {"type": "text", "text": "Result 3"},
101 + ],
102 + }
103 +
104 + messages = parse_input_to_harmony_message(chat_msg)
105 +
106 + assert len(messages) == 1
107 + assert messages[0].author.role == Role.TOOL
108 + assert messages[0].content[0].text == "Result 1: Result 2: Result 3"
109 +
110 + def test_tool_message_with_empty_content(self):
111 + """Test parsing tool message with None content."""
112 + chat_msg = {
113 + "role": "tool",
114 + "name": "empty_tool",
115 + "content": None,
116 + }
117 +
118 + messages = parse_input_to_harmony_message(chat_msg)
119 +
120 + assert len(messages) == 1
121 + assert messages[0].content[0].text == ""
122 +
123 + def test_system_message(self):
124 + """Test parsing system message."""
125 + chat_msg = {
126 + "role": "system",
127 + "content": "You are a helpful assistant",
128 + }
129 +
130 + messages = parse_input_to_harmony_message(chat_msg)
131 +
132 + assert len(messages) == 1
133 + # System messages are converted using Message.from_dict
134 + # which should preserve the role
135 + assert messages[0].author.role == Role.SYSTEM
136 +
137 + def test_developer_message(self):
138 + """Test parsing developer message."""
139 + chat_msg = {
140 + "role": "developer",
141 + "content": "Use concise language",
142 + }
143 +
144 + messages = parse_input_to_harmony_message(chat_msg)
145 +
146 + assert len(messages) == 1
147 + assert messages[0].author.role == Role.DEVELOPER
148 +
149 + def test_user_message_with_string_content(self):
150 + """Test parsing user message with string content."""
151 + chat_msg = {
152 + "role": "user",
153 + "content": "What's the weather in San Francisco?",
154 + }
155 +
156 + messages = parse_input_to_harmony_message(chat_msg)
157 +
158 + assert len(messages) == 1
159 + assert messages[0].author.role == Role.USER
160 + assert messages[0].content[0].text == "What's the weather in San Francisco?"
161 +
162 + def test_user_message_with_array_content(self):
163 + """Test parsing user message with array content."""
164 + chat_msg = {
165 + "role": "user",
166 + "content": [
167 + {"text": "What's in this image? "},
168 + {"text": "Please describe it."},
169 + ],
170 + }
171 +
172 + messages = parse_input_to_harmony_message(chat_msg)
173 +
174 + assert len(messages) == 1
175 + assert messages[0].author.role == Role.USER
176 + assert len(messages[0].content) == 2
177 + assert messages[0].content[0].text == "What's in this image? "
178 + assert messages[0].content[1].text == "Please describe it."
179 +
180 + def test_assistant_message_with_string_content(self):
181 + """Test parsing assistant message with string content (no tool calls)."""
182 + chat_msg = {
183 + "role": "assistant",
184 + "content": "Hello! How can I help you today?",
185 + }
186 +
187 + messages = parse_input_to_harmony_message(chat_msg)
188 +
189 + assert len(messages) == 1
190 + assert messages[0].author.role == Role.ASSISTANT
191 + assert messages[0].content[0].text == "Hello! How can I help you today?"
192 +
193 + def test_pydantic_model_input(self):
194 + """Test parsing Pydantic model input (has model_dump method)."""
195 +
196 + class MockPydanticModel:
197 + def model_dump(self, exclude_none=True):
198 + return {
199 + "role": "user",
200 + "content": "Test message",
201 + }
202 +
203 + chat_msg = MockPydanticModel()
204 + messages = parse_input_to_harmony_message(chat_msg)
205 +
206 + assert len(messages) == 1
207 + assert messages[0].author.role == Role.USER
208 + assert messages[0].content[0].text == "Test message"
209 +
210 + def test_message_with_empty_content(self):
211 + """Test parsing message with empty string content."""
212 + chat_msg = {
213 + "role": "user",
214 + "content": "",
215 + }
216 +
217 + messages = parse_input_to_harmony_message(chat_msg)
218 +
219 + assert len(messages) == 1
220 + assert messages[0].content[0].text == ""
221 +
222 + def test_tool_call_with_missing_function_fields(self):
223 + """Test parsing tool call with missing name or arguments."""
224 + chat_msg = {
225 + "role": "assistant",
226 + "tool_calls": [
227 + {
228 + "function": {} # Missing both name and arguments
229 + }
230 + ],
231 + }
232 +
233 + messages = parse_input_to_harmony_message(chat_msg)
234 +
235 + assert len(messages) == 1
236 + assert messages[0].recipient == "functions."
237 + assert messages[0].content[0].text == ""
238 +
239 + def test_array_content_with_missing_text(self):
240 + """Test parsing array content where text field is missing."""
241 + chat_msg = {
242 + "role": "user",
243 + "content": [
244 + {}, # Missing text field
245 + {"text": "actual text"},
246 + ],
247 + }
248 +
249 + messages = parse_input_to_harmony_message(chat_msg)
250 +
251 + assert len(messages) == 1
252 + assert len(messages[0].content) == 2
253 + assert messages[0].content[0].text == ""
254 + assert messages[0].content[1].text == "actual text"
vllm/entrypoints/harmony_utils.py
+38 -1
@@ -38,11 +38,14 @@ from openai_harmony import (
38 ToolDescription,
39 load_harmony_encoding,
40 )
41 +from openai_harmony import Message as OpenAIHarmonyMessage
42 +from openai_harmony import Role as OpenAIHarmonyRole
43
44 from vllm import envs
45 from vllm.entrypoints.openai.protocol import (
46 ChatCompletionToolsParam,
47 ResponseInputOutputItem,
48 + ResponsesRequest,
49 )
50 from vllm.utils import random_uuid
51
@@ -228,7 +231,7 @@ def parse_response_input(
231 return msg
232
233
231 -def parse_chat_input(chat_msg) -> list[Message]:
234 +def parse_input_to_harmony_message(chat_msg) -> list[Message]:
235 if not isinstance(chat_msg, dict):
236 # Handle Pydantic models
237 chat_msg = chat_msg.model_dump(exclude_none=True)
@@ -279,6 +282,40 @@ def parse_chat_input(chat_msg) -> list[Message]:
282 return [msg]
283
284
285 +def construct_harmony_previous_input_messages(
286 + request: ResponsesRequest,
287 +) -> list[OpenAIHarmonyMessage]:
288 + messages: list[OpenAIHarmonyMessage] = []
289 + if request.previous_input_messages:
290 + for message in request.previous_input_messages:
291 + # Handle both OpenAIHarmonyMessage objects and dictionary inputs
292 + if isinstance(message, OpenAIHarmonyMessage):
293 + message_role = message.author.role
294 + # To match OpenAI, instructions, reasoning and tools are
295 + # always taken from the most recent Responses API request
296 + # not carried over from previous requests
297 + if (
298 + message_role == OpenAIHarmonyRole.SYSTEM
299 + or message_role == OpenAIHarmonyRole.DEVELOPER
300 + ):
301 + continue
302 + messages.append(message)
303 + else:
304 + harmony_messages = parse_input_to_harmony_message(message)
305 + for harmony_msg in harmony_messages:
306 + message_role = harmony_msg.author.role
307 + # To match OpenAI, instructions, reasoning and tools are
308 + # always taken from the most recent Responses API request
309 + # not carried over from previous requests
310 + if (
311 + message_role == OpenAIHarmonyRole.SYSTEM
312 + or message_role == OpenAIHarmonyRole.DEVELOPER
313 + ):
314 + continue
315 + messages.append(harmony_msg)
316 + return messages
317 +
318 +
319 def render_for_completion(messages: list[Message]) -> list[int]:
320 conversation = Conversation.from_messages(messages)
321 token_ids = get_encoding().render_conversation_for_completion(
vllm/entrypoints/openai/protocol.py
+7 -1
@@ -47,6 +47,7 @@ from openai.types.responses import (
47 from openai.types.responses.response_reasoning_item import (
48 Content as ResponseReasoningTextContent,
49 )
50 +from openai_harmony import Message as OpenAIHarmonyMessage
51
52 from vllm.utils.serial_utils import (
53 EmbedDType,
@@ -383,10 +384,15 @@ class ResponsesRequest(OpenAIBaseModel):
384 default=False,
385 description=(
386 "Dictates whether or not to return messages as part of the "
386 - "response object. Currently only supported for non-streaming "
387 + "response object. Currently only supported for"
388 "non-background and gpt-oss only. "
389 ),
390 )
391 + # similar to input_messages / output_messages in ResponsesResponse
392 + # we take in previous_input_messages (ie in harmony format)
393 + # this cannot be used in conjunction with previous_response_id
394 + # TODO: consider supporting non harmony messages as well
395 + previous_input_messages: list[OpenAIHarmonyMessage | dict] | None = None
396 # --8<-- [end:responses-extra-params]
397
398 _DEFAULT_SAMPLING_PARAMS = {
vllm/entrypoints/openai/serving_chat.py
+21 -15
@@ -27,8 +27,8 @@ from vllm.entrypoints.harmony_utils import (
27 get_stop_tokens_for_assistant_actions,
28 get_streamable_parser_for_assistant,
29 get_system_message,
30 - parse_chat_input,
30 parse_chat_output,
31 + parse_input_to_harmony_message,
32 render_for_completion,
33 )
34 from vllm.entrypoints.logger import RequestLogger
@@ -1351,11 +1351,13 @@ class OpenAIServingChat(OpenAIServing):
1351 index=output.index,
1352 message=message,
1353 logprobs=logprobs,
1354 - finish_reason="tool_calls"
1355 - if (tool_call_info is not None and tool_call_info.tools_called)
1356 - else output.finish_reason
1357 - if output.finish_reason
1358 - else "stop",
1354 + finish_reason=(
1355 + "tool_calls"
1356 + if (tool_call_info is not None and tool_call_info.tools_called)
1357 + else output.finish_reason
1358 + if output.finish_reason
1359 + else "stop"
1360 + ),
1361 stop_reason=output.stop_reason,
1362 )
1363 choices.append(choice_data)
@@ -1522,11 +1524,13 @@ class OpenAIServingChat(OpenAIServing):
1524 index=output.index,
1525 message=message,
1526 logprobs=logprobs,
1525 - finish_reason="tool_calls"
1526 - if auto_tools_called
1527 - else output.finish_reason
1528 - if output.finish_reason
1529 - else "stop",
1527 + finish_reason=(
1528 + "tool_calls"
1529 + if auto_tools_called
1530 + else output.finish_reason
1531 + if output.finish_reason
1532 + else "stop"
1533 + ),
1534 stop_reason=output.stop_reason,
1535 token_ids=(
1536 as_list(output.token_ids) if request.return_token_ids else None
@@ -1685,9 +1689,11 @@ class OpenAIServingChat(OpenAIServing):
1689 should_return_as_token_id,
1690 ),
1691 logprob=max(step_token.logprob, -9999.0),
1688 - bytes=None
1689 - if step_decoded is None
1690 - else list(step_decoded.encode("utf-8", errors="replace")),
1692 + bytes=(
1693 + None
1694 + if step_decoded is None
1695 + else list(step_decoded.encode("utf-8", errors="replace"))
1696 + ),
1697 top_logprobs=self._get_top_logprobs(
1698 step_top_logprobs,
1699 num_output_top_logprobs,
@@ -1764,7 +1770,7 @@ class OpenAIServingChat(OpenAIServing):
1770
1771 # Add user message.
1772 for chat_msg in request.messages:
1767 - messages.extend(parse_chat_input(chat_msg))
1773 + messages.extend(parse_input_to_harmony_message(chat_msg))
1774
1775 # Render prompt token ids.
1776 prompt_token_ids = render_for_completion(messages)
vllm/entrypoints/openai/serving_responses.py
+10
@@ -63,6 +63,7 @@ from vllm.entrypoints.context import (
63 StreamingHarmonyContext,
64 )
65 from vllm.entrypoints.harmony_utils import (
66 + construct_harmony_previous_input_messages,
67 get_developer_message,
68 get_stop_tokens_for_assistant_actions,
69 get_system_message,
@@ -248,6 +249,13 @@ class OpenAIServingResponses(OpenAIServing):
249 ),
250 status_code=HTTPStatus.BAD_REQUEST,
251 )
252 + if request.previous_input_messages and request.previous_response_id:
253 + return self.create_error_response(
254 + err_type="invalid_request_error",
255 + message="Only one of `previous_input_messages` and "
256 + "`previous_response_id` can be set.",
257 + status_code=HTTPStatus.BAD_REQUEST,
258 + )
259 return None
260
261 async def create_responses(
@@ -941,6 +949,8 @@ class OpenAIServingResponses(OpenAIServing):
949 instructions=request.instructions, tools=request.tools
950 )
951 messages.append(dev_msg)
952 + messages += construct_harmony_previous_input_messages(request)
953 +
954 else:
955 # Continue the previous conversation.
956 # FIXME(woosuk): Currently, request params like reasoning and