A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 17:15 UTC
[responsesAPI] parse reasoning item input (#28248)
Signed-off-by: Andrew Xia <axia@fb.com> Co-authored-by: Andrew Xia <axia@fb.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Andrew Xia committed
Nov 21, 2025 at 23:42 UTC
742e9ff6b39ad0433bac0d7417a41bbdc74854a3
5 files changed
+212
-1
examples/online_serving/openai_responses_client.py
new
+44
@@ -0,0 +1,44 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+Set up this example by starting a vLLM OpenAI-compatible server.
5
+Reasoning models can be used through the Responses API as seen here
6
+https://platform.openai.com/docs/api-reference/responses
7
+For example:
8
+vllm serve Qwen/Qwen3-8B --reasoning-parser qwen3
9
+
10
+"""
11
+
12
+from openai import OpenAI
13
+
14
+input_messages = [{"role": "user", "content": "What model are you?"}]
15
+
16
+
17
+def main():
18
+ base_url = "http://localhost:8000/v1"
19
+ client = OpenAI(base_url=base_url, api_key="empty")
20
+ model = "Qwen/Qwen3-8B" # get_first_model(client)
21
+ response = client.responses.create(
22
+ model=model,
23
+ input=input_messages,
24
+ )
25
+
26
+ for message in response.output:
27
+ if message.type == "reasoning":
28
+ # append reasoning message
29
+ input_messages.append(message)
30
+
31
+ response_2 = client.responses.create(
32
+ model=model,
33
+ input=input_messages,
34
+ )
35
+ print(response_2.output_text)
36
+ # I am Qwen, a large language model developed by Alibaba Cloud.
37
+ # I am designed to assist with a wide range of tasks, including
38
+ # answering questions, creating content, coding, and engaging in
39
+ # conversations. I can help with various topics and provide
40
+ # information or support in multiple languages. How can I assist you today?
41
+
42
+
43
+if __name__ == "__main__":
44
+ main()
tests/entrypoints/openai/test_response_api_simple.py
new
+71
@@ -0,0 +1,71 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+import pytest
6
+import pytest_asyncio
7
+from openai import OpenAI
8
+
9
+from ...utils import RemoteOpenAIServer
10
+
11
+MODEL_NAME = "Qwen/Qwen3-8B"
12
+
13
+
14
+@pytest.fixture(scope="module")
15
+def server():
16
+ args = ["--reasoning-parser", "qwen3", "--max_model_len", "5000"]
17
+ env_dict = dict(
18
+ VLLM_ENABLE_RESPONSES_API_STORE="1",
19
+ # uncomment for tool calling
20
+ # PYTHON_EXECUTION_BACKEND="dangerously_use_uv",
21
+ )
22
+
23
+ with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
24
+ yield remote_server
25
+
26
+
27
+@pytest_asyncio.fixture
28
+async def client(server):
29
+ async with server.get_async_client() as async_client:
30
+ yield async_client
31
+
32
+
33
+@pytest.mark.asyncio
34
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
35
+async def test_basic(client: OpenAI, model_name: str):
36
+ response = await client.responses.create(
37
+ model=model_name,
38
+ input="What is 13 * 24?",
39
+ )
40
+ assert response is not None
41
+ print("response: ", response)
42
+ assert response.status == "completed"
43
+
44
+
45
+@pytest.mark.asyncio
46
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
47
+async def test_reasoning_item(client: OpenAI, model_name: str):
48
+ response = await client.responses.create(
49
+ model=model_name,
50
+ input=[
51
+ {"type": "message", "content": "Hello.", "role": "user"},
52
+ {
53
+ "type": "reasoning",
54
+ "id": "lol",
55
+ "content": [
56
+ {
57
+ "type": "reasoning_text",
58
+ "text": "We need to respond: greeting.",
59
+ }
60
+ ],
61
+ "summary": [],
62
+ },
63
+ ],
64
+ temperature=0.0,
65
+ )
66
+ assert response is not None
67
+ assert response.status == "completed"
68
+ # make sure we get a reasoning and text output
69
+ assert response.output[0].type == "reasoning"
70
+ assert response.output[1].type == "message"
71
+ assert type(response.output[1].content[0].text) is str
tests/entrypoints/openai/test_response_api_with_harmony.py
+26
-1
@@ -35,7 +35,7 @@ GET_WEATHER_SCHEMA = {
35
36
@pytest.fixture(scope="module")
37
def server():
38
- args = ["--enforce-eager", "--tool-server", "demo"]
38
+ args = ["--enforce-eager", "--tool-server", "demo", "--max_model_len", "5000"]
39
env_dict = dict(
40
VLLM_ENABLE_RESPONSES_API_STORE="1",
41
PYTHON_EXECUTION_BACKEND="dangerously_use_uv",
@@ -550,6 +550,31 @@ def call_function(name, args):
550
raise ValueError(f"Unknown function: {name}")
551
552
553
+@pytest.mark.asyncio
554
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
555
+async def test_reasoning_item(client: OpenAI, model_name: str):
556
+ response = await client.responses.create(
557
+ model=model_name,
558
+ input=[
559
+ {"type": "message", "content": "Hello.", "role": "user"},
560
+ {
561
+ "type": "reasoning",
562
+ "id": "lol",
563
+ "content": [
564
+ {
565
+ "type": "reasoning_text",
566
+ "text": "We need to respond: greeting.",
567
+ }
568
+ ],
569
+ "summary": [],
570
+ },
571
+ ],
572
+ temperature=0.0,
573
+ )
574
+ assert response is not None
575
+ assert response.status == "completed"
576
+
577
+
578
@pytest.mark.asyncio
579
@pytest.mark.parametrize("model_name", [MODEL_NAME])
580
async def test_function_calling(client: OpenAI, model_name: str):
tests/entrypoints/test_responses_utils.py
+58
@@ -1,7 +1,15 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import pytest
5
+from openai.types.responses.response_reasoning_item import (
6
+ Content,
7
+ ResponseReasoningItem,
8
+ Summary,
9
+)
10
+
11
from vllm.entrypoints.responses_utils import (
12
+ construct_chat_message_with_tool_call,
13
convert_tool_responses_to_completions_format,
14
)
15
@@ -28,3 +36,53 @@ class TestResponsesUtils:
36
result = convert_tool_responses_to_completions_format(input_tool)
37
38
assert result == {"type": "function", "function": input_tool}
39
+
40
+ def test_construct_chat_message_with_tool_call(self):
41
+ item = ResponseReasoningItem(
42
+ id="lol",
43
+ summary=[],
44
+ type="reasoning",
45
+ content=[
46
+ Content(
47
+ text="Leroy Jenkins",
48
+ type="reasoning_text",
49
+ )
50
+ ],
51
+ encrypted_content=None,
52
+ status=None,
53
+ )
54
+ formatted_item = construct_chat_message_with_tool_call(item)
55
+ assert formatted_item["role"] == "assistant"
56
+ assert formatted_item["reasoning"] == "Leroy Jenkins"
57
+
58
+ item = ResponseReasoningItem(
59
+ id="lol",
60
+ summary=[
61
+ Summary(
62
+ text='Hmm, the user has just started with a simple "Hello,"',
63
+ type="summary_text",
64
+ )
65
+ ],
66
+ type="reasoning",
67
+ content=None,
68
+ encrypted_content=None,
69
+ status=None,
70
+ )
71
+
72
+ formatted_item = construct_chat_message_with_tool_call(item)
73
+ assert formatted_item["role"] == "assistant"
74
+ assert (
75
+ formatted_item["reasoning"]
76
+ == 'Hmm, the user has just started with a simple "Hello,"'
77
+ )
78
+
79
+ item = ResponseReasoningItem(
80
+ id="lol",
81
+ summary=[],
82
+ type="reasoning",
83
+ content=None,
84
+ encrypted_content="TOP_SECRET_MESSAGE",
85
+ status=None,
86
+ )
87
+ with pytest.raises(ValueError):
88
+ construct_chat_message_with_tool_call(item)
vllm/entrypoints/responses_utils.py
+13
@@ -10,6 +10,7 @@ from openai.types.chat.chat_completion_message_tool_call_param import (
10
Function as FunctionCallTool,
11
)
12
from openai.types.responses import ResponseFunctionToolCall
13
+from openai.types.responses.response_reasoning_item import ResponseReasoningItem
14
from openai.types.responses.tool import Tool
15
16
from vllm import envs
@@ -37,6 +38,18 @@ def construct_chat_message_with_tool_call(
38
)
39
],
40
)
41
+ elif isinstance(item, ResponseReasoningItem):
42
+ reasoning_content = ""
43
+ if item.encrypted_content:
44
+ raise ValueError("Encrypted content is not supported.")
45
+ if len(item.summary) == 1:
46
+ reasoning_content = item.summary[0].text
47
+ elif item.content and len(item.content) == 1:
48
+ reasoning_content = item.content[0].text
49
+ return {
50
+ "role": "assistant",
51
+ "reasoning": reasoning_content,
52
+ }
53
elif item.get("type") == "function_call_output":
54
# Append the function call output as a tool message.
55
return ChatCompletionToolMessageParam(