A high-throughput and memory-efficient inference and serving engine for LLMs
[Frontend] Support strict mode for tool calling with ResponsesAPI (#45396)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Jun 12, 2026 at 22:59 UTC
3b8fc3fe6d4afe6680cfc96f5b15fccf4bfff46f
5 files changed
+98
-23
tests/entrypoints/openai/responses/conftest.py
-1
@@ -390,7 +390,6 @@ def server_with_store(default_server_args):
390
env_dict={
391
"VLLM_ENABLE_RESPONSES_API_STORE": "1",
392
"VLLM_SERVER_DEV_MODE": "1",
393
- "VLLM_ENFORCE_STRICT_TOOL_CALLING": "0",
393
},
394
) as remote_server:
395
yield remote_server
vllm/parser/abstract_parser.py
+9
-4
@@ -438,8 +438,7 @@ class DelegatingParser(Parser):
438
self, request: ChatCompletionRequest | ResponsesRequest
439
) -> ChatCompletionRequest | ResponsesRequest:
440
if (
441
- not isinstance(request, ChatCompletionRequest)
442
- or self._tool_parser is None
441
+ self._tool_parser is None
442
or self._tool_parser.structural_tag_model is None
443
or not request.tools
444
):
@@ -448,7 +447,10 @@ class DelegatingParser(Parser):
447
need_tool_calling = (
448
request.tool_choice == "auto"
449
or request.tool_choice == "required"
451
- or isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
450
+ or isinstance(
451
+ request.tool_choice,
452
+ (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction),
453
+ )
454
)
455
if not need_tool_calling:
456
return request
@@ -464,7 +466,10 @@ class DelegatingParser(Parser):
466
request.structured_outputs = StructuredOutputsParams(
467
structural_tag=structural_tag,
468
)
467
- request.response_format = None
469
+ if isinstance(request, ResponsesRequest):
470
+ request.text = None
471
+ else:
472
+ request.response_format = None
473
return request
474
475
def extract_reasoning_streaming(
vllm/reasoning/abs_reasoning_parsers.py
+1
-2
@@ -181,9 +181,8 @@ class ReasoningParser:
181
) -> str | None:
182
"""
183
Instance method that is implemented for preparing the structured tag
184
- Otherwise, None is returned
184
"""
186
- return None
185
+ return original_tag
186
187
188
class ReasoningParserManager:
vllm/tool_parsers/abstract_tool_parser.py
+4
-1
@@ -165,7 +165,10 @@ class ToolParser:
165
return request
166
167
def get_structural_tag(
168
- self, request: ChatCompletionRequest, *, reasoning: bool = False
168
+ self,
169
+ request: ChatCompletionRequest | ResponsesRequest,
170
+ *,
171
+ reasoning: bool = False,
172
):
173
if self.structural_tag_model is None:
174
return None
vllm/tool_parsers/structural_tag_registry.py
+84
-15
@@ -1,9 +1,14 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from collections.abc import Callable
5
-from typing import Any, Literal
6
-
4
+from collections.abc import Callable, Sequence
5
+from typing import Any, Literal, TypeAlias
6
+
7
+from openai.types.responses import FunctionTool
8
+from openai.types.responses.response import ToolChoice as ResponsesToolChoice
9
+from openai.types.responses.tool import Tool as ResponsesTool
10
+from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
11
+from openai.types.responses.tool_choice_function import ToolChoiceFunction
12
from xgrammar import StructuralTag, normalize_tool_choice
13
from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag
14
from xgrammar.openai_tool_call_schema import (
@@ -25,11 +30,15 @@ from vllm.entrypoints.openai.chat_completion.protocol import (
30
ChatCompletionToolsParam,
31
)
32
28
-ToolChoice = (
29
- Literal["none", "auto", "required"] | ChatCompletionNamedToolChoiceParam | None
33
+ToolChoice: TypeAlias = (
34
+ Literal["none", "auto", "required"]
35
+ | ChatCompletionNamedToolChoiceParam
36
+ | ResponsesToolChoice
37
+ | None
38
)
31
-SimplifiedToolChoice = Literal["auto", "required", "forced"]
32
-StructuralTagBuilder = Callable[
39
+AllowedToolRef: TypeAlias = dict[str, object]
40
+SimplifiedToolChoice: TypeAlias = Literal["auto", "required", "forced"]
41
+StructuralTagBuilder: TypeAlias = Callable[
42
[
43
list[FunctionToolParam],
44
list[BuiltinToolParam],
@@ -77,7 +86,7 @@ def register_vllm_structural_tag(model: str):
86
87
def get_model_structural_tag(
88
model: str,
80
- tools: list[ChatCompletionToolsParam] | None,
89
+ tools: Sequence[ChatCompletionToolsParam | ResponsesTool] | None,
90
tool_choice: ToolChoice,
91
reasoning: bool,
92
) -> StructuralTag | None:
@@ -86,8 +95,8 @@ def get_model_structural_tag(
95
if not tools or tool_choice == "none":
96
return None
97
89
- dumped_tools = [_model_dump(tool) for tool in tools]
90
- dumped_tool_choice = _model_dump(tool_choice)
98
+ dumped_tools = [_dump_tool_for_xgrammar(tool) for tool in tools]
99
+ dumped_tool_choice = _dump_tool_choice_for_xgrammar(tool_choice)
100
101
if model in _VLLM_STRUCTURAL_TAG_REGISTRY:
102
function_tools, builtin_tools, simplified_tool_choice = normalize_tool_choice(
@@ -113,12 +122,72 @@ def get_model_structural_tag(
122
)
123
124
116
-def _model_dump(value: Any) -> Any:
117
- """Convert vLLM/Pydantic request objects to xgrammar's dict protocol."""
125
+def _dump_tool_for_xgrammar(
126
+ tool: ChatCompletionToolsParam | ResponsesTool,
127
+) -> dict[str, Any]:
128
+ """Convert tool objects to xgrammar's Chat Completions tool protocol."""
129
+
130
+ if isinstance(tool, FunctionTool):
131
+ function: dict[str, Any] = {"name": tool.name}
132
+ if tool.description is not None:
133
+ function["description"] = tool.description
134
+ if tool.parameters is not None:
135
+ function["parameters"] = tool.parameters
136
+ if tool.strict is not None:
137
+ function["strict"] = tool.strict
138
+ return {"type": "function", "function": function}
139
+ dumped_tool = tool.model_dump(mode="json", exclude_none=True)
140
+ if isinstance(tool, ChatCompletionToolsParam):
141
+ return dumped_tool
142
+ return dict(dumped_tool)
143
+
144
+
145
+def _dump_tool_choice_for_xgrammar(
146
+ tool_choice: ToolChoice,
147
+) -> dict[str, Any] | str | None:
148
+ """Convert tool_choice objects to xgrammar's expected protocol."""
149
+
150
+ if tool_choice is None:
151
+ return None
152
119
- if hasattr(value, "model_dump"):
120
- return value.model_dump(exclude_none=True)
121
- return value
153
+ if isinstance(tool_choice, str):
154
+ return tool_choice
155
+
156
+ if isinstance(tool_choice, ChatCompletionNamedToolChoiceParam):
157
+ return tool_choice.model_dump(mode="json", exclude_none=True)
158
+
159
+ if isinstance(tool_choice, ToolChoiceFunction):
160
+ return {
161
+ "type": "function",
162
+ "function": {"name": tool_choice.name},
163
+ }
164
+
165
+ if isinstance(tool_choice, ToolChoiceAllowed):
166
+ return {
167
+ "type": "allowed_tools",
168
+ "allowed_tools": {
169
+ "mode": tool_choice.mode,
170
+ "tools": [
171
+ _dump_allowed_tool_ref_for_xgrammar(tool)
172
+ for tool in tool_choice.tools
173
+ ],
174
+ },
175
+ }
176
+
177
+ return tool_choice.model_dump(mode="json", exclude_none=True)
178
+
179
+
180
+def _dump_allowed_tool_ref_for_xgrammar(tool_ref: AllowedToolRef) -> AllowedToolRef:
181
+ if (
182
+ tool_ref.get("type") == "function"
183
+ and "function" not in tool_ref
184
+ and "name" in tool_ref
185
+ ):
186
+ return {
187
+ "type": "function",
188
+ "function": {"name": tool_ref["name"]},
189
+ }
190
+ return tool_ref
191
192
193
def _get_function_parameters(function) -> dict[str, Any] | bool: