A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 04:45 UTC
[responsesAPI][3] ResponsesParser to set up non harmony MCP (#29413)
Signed-off-by: Andrew Xia <axia@fb.com> Co-authored-by: Andrew Xia <axia@fb.com>
Andrew Xia committed
Dec 2, 2025 at 08:24 UTC
52cb349fc010c3d9e8f576f7cc675e6403aadd0a
8 files changed
+332
-13
tests/entrypoints/openai/test_response_api_parsable_context.py
new
+87
@@ -0,0 +1,87 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+import pytest
6
+import pytest_asyncio
7
+from openai import OpenAI
8
+
9
+from ...utils import RemoteOpenAIServer
10
+
11
+MODEL_NAME = "Qwen/Qwen3-8B"
12
+
13
+
14
+@pytest.fixture(scope="module")
15
+def server():
16
+ args = ["--reasoning-parser", "qwen3", "--max_model_len", "5000"]
17
+ env_dict = dict(
18
+ VLLM_ENABLE_RESPONSES_API_STORE="1",
19
+ VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT="1",
20
+ # uncomment for tool calling
21
+ # PYTHON_EXECUTION_BACKEND="dangerously_use_uv",
22
+ )
23
+
24
+ with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
25
+ yield remote_server
26
+
27
+
28
+@pytest_asyncio.fixture
29
+async def client(server):
30
+ async with server.get_async_client() as async_client:
31
+ yield async_client
32
+
33
+
34
+@pytest.mark.asyncio
35
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
36
+async def test_basic(client: OpenAI, model_name: str):
37
+ response = await client.responses.create(
38
+ model=model_name,
39
+ input="What is 13 * 24?",
40
+ )
41
+ assert response is not None
42
+ print("response: ", response)
43
+ assert response.status == "completed"
44
+
45
+
46
+@pytest.mark.asyncio
47
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
48
+async def test_reasoning_and_function_items(client: OpenAI, model_name: str):
49
+ response = await client.responses.create(
50
+ model=model_name,
51
+ input=[
52
+ {"type": "message", "content": "Hello.", "role": "user"},
53
+ {
54
+ "type": "reasoning",
55
+ "id": "lol",
56
+ "content": [
57
+ {
58
+ "type": "reasoning_text",
59
+ "text": "We need to respond: greeting.",
60
+ }
61
+ ],
62
+ "summary": [],
63
+ },
64
+ {
65
+ "arguments": '{"location": "Paris", "unit": "celsius"}',
66
+ "call_id": "call_5f7b38f3b81e4b8380fd0ba74f3ca3ab",
67
+ "name": "get_weather",
68
+ "type": "function_call",
69
+ "id": "fc_4fe5d6fc5b6c4d6fa5f24cc80aa27f78",
70
+ "status": "completed",
71
+ },
72
+ {
73
+ "call_id": "call_5f7b38f3b81e4b8380fd0ba74f3ca3ab",
74
+ "id": "fc_4fe5d6fc5b6c4d6fa5f24cc80aa27f78",
75
+ "output": "The weather in Paris is 20 Celsius",
76
+ "status": "completed",
77
+ "type": "function_call_output",
78
+ },
79
+ ],
80
+ temperature=0.0,
81
+ )
82
+ assert response is not None
83
+ assert response.status == "completed"
84
+ # make sure we get a reasoning and text output
85
+ assert response.output[0].type == "reasoning"
86
+ assert response.output[1].type == "message"
87
+ assert type(response.output[1].content[0].text) is str
vllm/entrypoints/chat_utils.py
+1
@@ -1530,6 +1530,7 @@ def _parse_chat_message_content(
1530
role = message["role"]
1531
content = message.get("content")
1532
reasoning = message.get("reasoning") or message.get("reasoning_content")
1533
+
1534
if content is None:
1535
content = []
1536
elif isinstance(content, str):
vllm/entrypoints/context.py
+76
@@ -5,6 +5,7 @@ import contextlib
5
import json
6
import logging
7
from abc import ABC, abstractmethod
8
+from collections.abc import Callable
9
from contextlib import AsyncExitStack
10
from typing import TYPE_CHECKING, Union
11
@@ -17,9 +18,19 @@ from vllm.entrypoints.harmony_utils import (
18
get_streamable_parser_for_assistant,
19
render_for_completion,
20
)
21
+from vllm.entrypoints.openai.parser.responses_parser import (
22
+ get_responses_parser_for_simple_context,
23
+)
24
+from vllm.entrypoints.openai.protocol import (
25
+ ResponseInputOutputItem,
26
+ ResponsesRequest,
27
+)
28
+from vllm.entrypoints.responses_utils import construct_tool_dicts
29
from vllm.entrypoints.tool import Tool
30
from vllm.entrypoints.tool_server import ToolServer
31
from vllm.outputs import RequestOutput
32
+from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
33
+from vllm.transformers_utils.tokenizer import AnyTokenizer
34
35
if TYPE_CHECKING:
36
from mcp.client import ClientSession
@@ -180,6 +191,71 @@ class SimpleContext(ConversationContext):
191
raise NotImplementedError("Should not be called.")
192
193
194
+class ParsableContext(ConversationContext):
195
+ def __init__(
196
+ self,
197
+ *,
198
+ response_messages: list[ResponseInputOutputItem],
199
+ tokenizer: AnyTokenizer,
200
+ reasoning_parser_cls: Callable[[AnyTokenizer], ReasoningParser] | None,
201
+ request: ResponsesRequest,
202
+ ):
203
+ self.num_prompt_tokens = 0
204
+ self.num_output_tokens = 0
205
+ self.num_cached_tokens = 0
206
+ # TODO: num_reasoning_tokens is not implemented yet.
207
+ self.num_reasoning_tokens = 0
208
+ # not implemented yet for ParsableContext
209
+ self.all_turn_metrics: list[TurnMetrics] = []
210
+
211
+ if reasoning_parser_cls is None:
212
+ raise ValueError("reasoning_parser_cls must be provided.")
213
+
214
+ self.parser = get_responses_parser_for_simple_context(
215
+ tokenizer=tokenizer,
216
+ reasoning_parser_cls=reasoning_parser_cls,
217
+ response_messages=response_messages,
218
+ request=request,
219
+ )
220
+
221
+ self._tool_sessions: dict[str, ClientSession | Tool] = {}
222
+ self.called_tools: set[str] = set()
223
+
224
+ self.tool_dicts = construct_tool_dicts(request.tools, request.tool_choice)
225
+
226
+ def append_output(self, output: RequestOutput) -> None:
227
+ self.num_prompt_tokens = len(output.prompt_token_ids or [])
228
+ self.num_cached_tokens = output.num_cached_tokens or 0
229
+ self.num_output_tokens += len(output.outputs[0].token_ids or [])
230
+ self.parser.process(output.outputs[0])
231
+
232
+ def append_tool_output(self, output: list[ResponseInputOutputItem]) -> None:
233
+ raise NotImplementedError("Should not be called.")
234
+
235
+ def need_builtin_tool_call(self) -> bool:
236
+ """Return true if the last message is a MCP tool call"""
237
+ return False
238
+
239
+ async def call_tool(self) -> list[ResponseInputOutputItem]:
240
+ raise NotImplementedError("Should not be called.")
241
+
242
+ def render_for_completion(self):
243
+ raise NotImplementedError("Should not be called.")
244
+
245
+ async def init_tool_sessions(
246
+ self,
247
+ tool_server: ToolServer | None,
248
+ exit_stack: AsyncExitStack,
249
+ request_id: str,
250
+ mcp_tools: dict[str, Mcp],
251
+ ):
252
+ pass
253
+
254
+ async def cleanup_session(self, *args, **kwargs) -> None:
255
+ """Can be used as coro to used in __aexit__"""
256
+ raise NotImplementedError("Should not be called.")
257
+
258
+
259
class HarmonyContext(ConversationContext):
260
def __init__(
261
self,
vllm/entrypoints/openai/parser/__init__.py
vllm/entrypoints/openai/parser/responses_parser.py
new
+101
@@ -0,0 +1,101 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import logging
4
+from collections.abc import Callable
5
+
6
+from openai.types.responses.response_output_message import ResponseOutputMessage
7
+from openai.types.responses.response_output_text import ResponseOutputText
8
+from openai.types.responses.response_reasoning_item import (
9
+ Content,
10
+ ResponseReasoningItem,
11
+)
12
+
13
+from vllm.entrypoints.openai.protocol import ResponseInputOutputItem, ResponsesRequest
14
+from vllm.outputs import CompletionOutput
15
+from vllm.reasoning.abs_reasoning_parsers import ReasoningParser
16
+from vllm.transformers_utils.tokenizer import AnyTokenizer
17
+from vllm.utils import random_uuid
18
+
19
+logger = logging.getLogger(__name__)
20
+
21
+
22
+class ResponsesParser:
23
+ """Incremental parser over completion tokens with reasoning support."""
24
+
25
+ def __init__(
26
+ self,
27
+ *,
28
+ tokenizer: AnyTokenizer,
29
+ reasoning_parser_cls: Callable[[AnyTokenizer], ReasoningParser],
30
+ response_messages: list[ResponseInputOutputItem],
31
+ request: ResponsesRequest,
32
+ ):
33
+ self.response_messages: list[ResponseInputOutputItem] = (
34
+ # TODO: initial messages may not be properly typed
35
+ response_messages
36
+ )
37
+ self.num_init_messages = len(response_messages)
38
+ self.tokenizer = tokenizer
39
+ self.request = request
40
+
41
+ self.reasoning_parser_instance = reasoning_parser_cls(tokenizer)
42
+
43
+ def process(self, output: CompletionOutput) -> "ResponsesParser":
44
+ reasoning_content, content = self.reasoning_parser_instance.extract_reasoning(
45
+ output.text, request=self.request
46
+ )
47
+ if reasoning_content:
48
+ self.response_messages.append(
49
+ ResponseReasoningItem(
50
+ type="reasoning",
51
+ id=f"rs_{random_uuid()}",
52
+ summary=[],
53
+ content=[
54
+ Content(
55
+ type="reasoning_text",
56
+ text=reasoning_content,
57
+ )
58
+ ],
59
+ )
60
+ )
61
+
62
+ if content:
63
+ self.response_messages.append(
64
+ ResponseOutputMessage(
65
+ type="message",
66
+ id=f"msg_{random_uuid()}",
67
+ status="completed",
68
+ role="assistant",
69
+ content=[
70
+ ResponseOutputText(
71
+ annotations=[], # TODO
72
+ type="output_text",
73
+ text=content,
74
+ logprobs=None, # TODO
75
+ )
76
+ ],
77
+ )
78
+ )
79
+
80
+ return self
81
+
82
+
83
+def get_responses_parser_for_simple_context(
84
+ *,
85
+ tokenizer: AnyTokenizer,
86
+ reasoning_parser_cls: Callable[[AnyTokenizer], ReasoningParser],
87
+ response_messages: list[ResponseInputOutputItem],
88
+ request: ResponsesRequest,
89
+) -> ResponsesParser:
90
+ """Factory function to create a ResponsesParser with
91
+ optional reasoning parser.
92
+
93
+ Returns:
94
+ ResponsesParser instance configured with the provided parser
95
+ """
96
+ return ResponsesParser(
97
+ tokenizer=tokenizer,
98
+ reasoning_parser_cls=reasoning_parser_cls,
99
+ response_messages=response_messages,
100
+ request=request,
101
+ )
vllm/entrypoints/openai/serving_responses.py
+32
-13
@@ -60,6 +60,7 @@ from vllm.entrypoints.chat_utils import (
60
from vllm.entrypoints.context import (
61
ConversationContext,
62
HarmonyContext,
63
+ ParsableContext,
64
SimpleContext,
65
StreamingHarmonyContext,
66
)
@@ -96,8 +97,9 @@ from vllm.entrypoints.openai.serving_engine import OpenAIServing
97
from vllm.entrypoints.openai.serving_models import OpenAIServingModels
98
from vllm.entrypoints.responses_utils import (
99
construct_input_messages,
99
- convert_tool_responses_to_completions_format,
100
+ construct_tool_dicts,
101
extract_tool_types,
102
+ make_response_output_items_from_parsable_context,
103
)
104
from vllm.entrypoints.tool_server import ToolServer
105
from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
@@ -228,7 +230,6 @@ class OpenAIServingResponses(OpenAIServing):
230
self.tool_parser = self._get_tool_parser(
231
tool_parser_name=tool_parser, enable_auto_tools=enable_auto_tools
232
)
231
- self.exclude_tools_when_tool_choice_none = False
233
# HACK(woosuk): This is a hack. We should use a better store.
234
# FIXME: If enable_store=True, this may cause a memory leak since we
235
# never remove responses from the store.
@@ -413,7 +414,17 @@ class OpenAIServingResponses(OpenAIServing):
414
else:
415
context = HarmonyContext(messages, available_tools)
416
else:
416
- context = SimpleContext()
417
+ if envs.VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT:
418
+ # This is an feature in development for parsing
419
+ # tokens during generation instead of at the end
420
+ context = ParsableContext(
421
+ response_messages=messages,
422
+ tokenizer=tokenizer,
423
+ reasoning_parser_cls=self.reasoning_parser,
424
+ request=request,
425
+ )
426
+ else:
427
+ context = SimpleContext()
428
429
if self.reasoning_parser is not None:
430
reasoning_parser = self.reasoning_parser(tokenizer)
@@ -534,15 +545,7 @@ class OpenAIServingResponses(OpenAIServing):
545
prev_response: ResponsesResponse | None,
546
tokenizer: TokenizerLike,
547
):
537
- if request.tools is None or (
538
- request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none
539
- ):
540
- tool_dicts = None
541
- else:
542
- tool_dicts = [
543
- convert_tool_responses_to_completions_format(tool.model_dump())
544
- for tool in request.tools
545
- ]
548
+ tool_dicts = construct_tool_dicts(request.tools, request.tool_choice)
549
# Construct the input messages.
550
messages = construct_input_messages(
551
request_instructions=request.instructions,
@@ -642,6 +645,22 @@ class OpenAIServingResponses(OpenAIServing):
645
status = "cancelled"
646
else:
647
status = "incomplete"
648
+ elif isinstance(context, ParsableContext):
649
+ response_messages = context.parser.response_messages[
650
+ context.parser.num_init_messages :
651
+ ]
652
+ output = make_response_output_items_from_parsable_context(response_messages)
653
+
654
+ # TODO: context for non-gptoss models doesn't use messages
655
+ # so we can't get them out yet
656
+ if request.enable_response_messages:
657
+ raise NotImplementedError(
658
+ "enable_response_messages is currently only supported for gpt-oss"
659
+ )
660
+
661
+ # TODO: Calculate usage.
662
+ # assert final_res.prompt_token_ids is not None
663
+ num_tool_output_tokens = 0
664
else:
665
assert isinstance(context, SimpleContext)
666
final_res = context.last_output
@@ -661,7 +680,7 @@ class OpenAIServingResponses(OpenAIServing):
680
assert final_res.prompt_token_ids is not None
681
num_tool_output_tokens = 0
682
664
- assert isinstance(context, (SimpleContext, HarmonyContext))
683
+ assert isinstance(context, (SimpleContext, HarmonyContext, ParsableContext))
684
num_prompt_tokens = context.num_prompt_tokens
685
num_generated_tokens = context.num_output_tokens
686
num_cached_tokens = context.num_cached_tokens
vllm/entrypoints/responses_utils.py
+30
@@ -1,6 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from typing import Any
5
+
6
from openai.types.chat import (
7
ChatCompletionAssistantMessageParam,
8
ChatCompletionMessageToolCallParam,
@@ -10,6 +12,7 @@ from openai.types.chat.chat_completion_message_tool_call_param import (
12
Function as FunctionCallTool,
13
)
14
from openai.types.responses import ResponseFunctionToolCall, ResponseOutputItem
15
+from openai.types.responses.response import ToolChoice
16
from openai.types.responses.response_function_tool_call_output_item import (
17
ResponseFunctionToolCallOutputItem,
18
)
@@ -24,6 +27,20 @@ from vllm.entrypoints.openai.protocol import (
27
)
28
29
30
+def make_response_output_items_from_parsable_context(
31
+ response_messages: list[ResponseInputOutputItem],
32
+) -> list[ResponseOutputItem]:
33
+ """Given a list of sentences, construct ResponseOutput Items."""
34
+ output_messages: list[ResponseOutputItem] = []
35
+ for message in response_messages:
36
+ if not isinstance(message, ResponseFunctionToolCallOutputItem):
37
+ output_messages.append(message)
38
+ else:
39
+ raise NotImplementedError("tool calls not supported for response context")
40
+
41
+ return output_messages
42
+
43
+
44
def construct_input_messages(
45
*,
46
request_instructions: str | None = None,
@@ -146,3 +163,16 @@ def convert_tool_responses_to_completions_format(tool: dict) -> dict:
163
"type": "function",
164
"function": tool,
165
}
166
+
167
+
168
+def construct_tool_dicts(
169
+ tools: list[Tool], tool_choice: ToolChoice
170
+) -> list[dict[str, Any]] | None:
171
+ if tools is None or (tool_choice == "none"):
172
+ tool_dicts = None
173
+ else:
174
+ tool_dicts = [
175
+ convert_tool_responses_to_completions_format(tool.model_dump())
176
+ for tool in tools
177
+ ]
178
+ return tool_dicts
vllm/envs.py
+5
@@ -214,6 +214,7 @@ if TYPE_CHECKING:
214
VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
215
VLLM_TUNED_CONFIG_FOLDER: str | None = None
216
VLLM_GPT_OSS_SYSTEM_TOOL_MCP_LABELS: set[str] = set()
217
+ VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT: bool = False
218
VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False
219
VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: bool = False
220
VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False
@@ -1444,6 +1445,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
1445
"VLLM_ALLREDUCE_USE_SYMM_MEM": lambda: bool(
1446
int(os.getenv("VLLM_ALLREDUCE_USE_SYMM_MEM", "1"))
1447
),
1448
+ # Experimental: use this to enable MCP tool calling for non harmony models
1449
+ "VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT": lambda: bool(
1450
+ int(os.getenv("VLLM_USE_EXPERIMENTAL_PARSER_CONTEXT", "0"))
1451
+ ),
1452
# Allows vllm to find tuned config under customized folder
1453
"VLLM_TUNED_CONFIG_FOLDER": lambda: os.getenv("VLLM_TUNED_CONFIG_FOLDER", None),
1454
# Valid values are container,code_interpreter,web_search_preview