A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 07:30 UTC
feat: update xgrammar==0.2.0 to use structural tags for strict tool calling + reasoning for more models (#40894)
Signed-off-by: Yuchuan <yuchuan.7streams@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Ubospica <ubospica@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Ubospica <ubospica@gmail.com> Co-authored-by: sfeng33 <4florafeng@gmail.com>
Linzhang Li committed
May 5, 2026 at 03:45 UTC
844df542694089045589ceaad52cba69ab58527a
10 files changed
+613
-6
requirements/common.txt
+1
-1
@@ -24,7 +24,7 @@ outlines_core == 0.2.14
24
# required for outlines backend disk cache
25
diskcache == 5.6.3
26
lark == 1.2.2
27
-xgrammar >= 0.1.32, < 1.0.0; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le"
27
+xgrammar >= 0.2.0, < 1.0.0; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le"
28
typing_extensions >= 4.10
29
filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
30
partial-json-parser # used for parsing partial JSON outputs
requirements/test/rocm.txt
+4
-1
@@ -42,6 +42,8 @@ anyio==4.13.0
42
# sse-starlette
43
# starlette
44
# watchfiles
45
+apache-tvm-ffi==0.1.10
46
+ # via xgrammar
47
arctic-inference==0.1.1
48
# via -r requirements/test/rocm.in
49
argcomplete==3.6.3
@@ -1264,6 +1266,7 @@ typing-extensions==4.15.0
1266
# alembic
1267
# anthropic
1268
# anyio
1269
+ # apache-tvm-ffi
1270
# azure-core
1271
# azure-identity
1272
# azure-storage-blob
@@ -1345,7 +1348,7 @@ word2number==1.1
1348
# via lm-eval
1349
wrapt==2.1.2
1350
# via smart-open
1348
-xgrammar==0.1.33
1351
+xgrammar==0.2.0
1352
# via
1353
# -c requirements/common.txt
1354
# -r requirements/test/../common.txt
tests/tool_parsers/test_deepseekv4_tool_parser.py
+82
@@ -6,6 +6,15 @@
6
import json
7
from unittest.mock import MagicMock
8
9
+import pytest
10
+from xgrammar import StructuralTag
11
+
12
+from vllm.entrypoints.openai.chat_completion.protocol import (
13
+ ChatCompletionNamedFunction,
14
+ ChatCompletionNamedToolChoiceParam,
15
+ ChatCompletionRequest,
16
+ ChatCompletionToolsParam,
17
+)
18
from vllm.tool_parsers import ToolParserManager
19
from vllm.tool_parsers.deepseekv4_tool_parser import DeepSeekV4ToolParser
20
@@ -20,6 +29,43 @@ PARAM_START = '<｜DSML｜parameter name="'
29
PARAM_END = "</｜DSML｜parameter>"
30
31
32
+@pytest.fixture
33
+def sample_tools() -> list[ChatCompletionToolsParam]:
34
+ return [
35
+ ChatCompletionToolsParam(
36
+ type="function",
37
+ function={
38
+ "name": "get_current_weather",
39
+ "description": "Get the current weather",
40
+ "parameters": {
41
+ "type": "object",
42
+ "properties": {
43
+ "city": {"type": "string", "description": "The city name"},
44
+ "state": {"type": "string", "description": "The state code"},
45
+ "unit": {"type": "string", "enum": ["fahrenheit", "celsius"]},
46
+ },
47
+ "required": ["city", "state"],
48
+ },
49
+ },
50
+ ),
51
+ ChatCompletionToolsParam(
52
+ type="function",
53
+ function={
54
+ "name": "calculate_area",
55
+ "description": "Calculate area of a shape",
56
+ "parameters": {
57
+ "type": "object",
58
+ "properties": {
59
+ "shape": {"type": "string"},
60
+ "dimensions": {"type": "object"},
61
+ "precision": {"type": "integer"},
62
+ },
63
+ },
64
+ },
65
+ ),
66
+ ]
67
+
68
+
69
def make_parser(tools=None) -> DeepSeekV4ToolParser:
70
return DeepSeekV4ToolParser(MOCK_TOKENIZER, tools=tools)
71
@@ -121,3 +167,39 @@ def test_streaming_extracts_complete_invokes():
167
]
168
assert names == ["search"]
169
assert json.loads(reconstruct_args(deltas)) == {"query": "deepseek v4"}
170
+
171
+
172
+def test_get_vllm_registry_structural_tag_returns_structural_tag(
173
+ sample_tools: list[ChatCompletionToolsParam],
174
+) -> None:
175
+ parser = make_parser()
176
+ req = ChatCompletionRequest(
177
+ messages=[],
178
+ model="m",
179
+ tools=sample_tools,
180
+ tool_choice="auto",
181
+ )
182
+ tag = parser.get_structural_tag(req)
183
+ assert isinstance(tag, StructuralTag)
184
+
185
+ req = ChatCompletionRequest(
186
+ messages=[],
187
+ model="m",
188
+ tools=sample_tools,
189
+ tool_choice="required",
190
+ )
191
+ tag = parser.get_structural_tag(req)
192
+ assert isinstance(tag, StructuralTag)
193
+
194
+ if sample_tools:
195
+ tool = sample_tools[0]
196
+ req = ChatCompletionRequest(
197
+ messages=[],
198
+ model="m",
199
+ tools=sample_tools,
200
+ )
201
+ req.tool_choice = ChatCompletionNamedToolChoiceParam(
202
+ function=ChatCompletionNamedFunction(name=tool.function.name)
203
+ )
204
+ tag = parser.get_structural_tag(req)
205
+ assert isinstance(tag, StructuralTag)
tests/tool_parsers/test_qwen3coder_tool_parser.py
+109
@@ -6,8 +6,11 @@ from collections.abc import Generator
6
7
import pytest
8
from openai.types.responses.function_tool import FunctionTool
9
+from xgrammar import StructuralTag
10
11
from vllm.entrypoints.openai.chat_completion.protocol import (
12
+ ChatCompletionNamedFunction,
13
+ ChatCompletionNamedToolChoiceParam,
14
ChatCompletionRequest,
15
ChatCompletionToolsParam,
16
)
@@ -108,6 +111,27 @@ def sample_tools(request):
111
]
112
113
114
+def _as_chat_completion_tools(
115
+ tools: list[ChatCompletionToolsParam | FunctionTool],
116
+) -> list[ChatCompletionToolsParam]:
117
+ normalized: list[ChatCompletionToolsParam] = []
118
+ for tool in tools:
119
+ if isinstance(tool, ChatCompletionToolsParam):
120
+ normalized.append(tool)
121
+ else:
122
+ normalized.append(
123
+ ChatCompletionToolsParam(
124
+ type="function",
125
+ function={
126
+ "name": tool.name,
127
+ "description": tool.description,
128
+ "parameters": tool.parameters,
129
+ },
130
+ )
131
+ )
132
+ return normalized
133
+
134
+
135
def assert_tool_calls(
136
actual_tool_calls: list[ToolCall], expected_tool_calls: list[ToolCall]
137
):
@@ -1146,3 +1170,88 @@ def test_no_double_serialization_string_args(qwen3_tool_parser):
1170
args = json.loads(raw_arguments)
1171
assert args["message"] == "hello world"
1172
assert '\\"hello world\\"' not in raw_arguments
1173
+
1174
+
1175
+def test_get_vllm_registry_structural_tag_returns_structural_tag(
1176
+ qwen3_tool_parser: Qwen3CoderToolParser,
1177
+ sample_tools: list[ChatCompletionToolsParam],
1178
+) -> None:
1179
+ request_tools = _as_chat_completion_tools(sample_tools)
1180
+ req = ChatCompletionRequest(
1181
+ messages=[],
1182
+ model="m",
1183
+ tools=request_tools,
1184
+ tool_choice="auto",
1185
+ )
1186
+ tag = qwen3_tool_parser.get_structural_tag(req)
1187
+ assert isinstance(tag, StructuralTag)
1188
+
1189
+ req = ChatCompletionRequest(
1190
+ messages=[],
1191
+ model="m",
1192
+ tools=request_tools,
1193
+ tool_choice="required",
1194
+ )
1195
+ tag = qwen3_tool_parser.get_structural_tag(req)
1196
+ assert isinstance(tag, StructuralTag)
1197
+
1198
+ if request_tools:
1199
+ tool = request_tools[0]
1200
+ req = ChatCompletionRequest(
1201
+ messages=[],
1202
+ model="m",
1203
+ tools=request_tools,
1204
+ )
1205
+ req.tool_choice = ChatCompletionNamedToolChoiceParam(
1206
+ function=ChatCompletionNamedFunction(name=tool.function.name)
1207
+ )
1208
+ tag = qwen3_tool_parser.get_structural_tag(req)
1209
+ assert isinstance(tag, StructuralTag)
1210
+
1211
+
1212
+@pytest.mark.parametrize("include_reasoning", [True, False])
1213
+def test_adjust_request_auto_uses_vllm_registry_structural_tag(
1214
+ monkeypatch: pytest.MonkeyPatch,
1215
+ qwen3_tool_parser: Qwen3CoderToolParser,
1216
+ sample_tools: list[ChatCompletionToolsParam],
1217
+ include_reasoning: bool,
1218
+) -> None:
1219
+ monkeypatch.setattr(
1220
+ "vllm.tool_parsers.abstract_tool_parser.VLLM_ENFORCE_STRICT_TOOL_CALLING",
1221
+ True,
1222
+ )
1223
+ request_tools = _as_chat_completion_tools(sample_tools)
1224
+ req = ChatCompletionRequest(
1225
+ messages=[],
1226
+ model="m",
1227
+ tools=request_tools,
1228
+ tool_choice="auto",
1229
+ include_reasoning=include_reasoning,
1230
+ )
1231
+ out = qwen3_tool_parser.adjust_request(req)
1232
+ assert out.structured_outputs is not None
1233
+ assert out.structured_outputs.structural_tag is not None
1234
+ assert isinstance(out.structured_outputs.structural_tag, str)
1235
+ loaded = json.loads(out.structured_outputs.structural_tag)
1236
+ assert isinstance(loaded, dict)
1237
+
1238
+
1239
+def test_adjust_request_required_prefers_structural_tag(
1240
+ monkeypatch: pytest.MonkeyPatch,
1241
+ qwen3_tool_parser: Qwen3CoderToolParser,
1242
+ sample_tools: list[ChatCompletionToolsParam],
1243
+) -> None:
1244
+ monkeypatch.setattr(
1245
+ "vllm.tool_parsers.abstract_tool_parser.VLLM_ENFORCE_STRICT_TOOL_CALLING",
1246
+ True,
1247
+ )
1248
+ request_tools = _as_chat_completion_tools(sample_tools)
1249
+ req = ChatCompletionRequest(
1250
+ messages=[],
1251
+ model="m",
1252
+ tools=request_tools,
1253
+ tool_choice="required",
1254
+ )
1255
+ out = qwen3_tool_parser.adjust_request(req)
1256
+ assert out.structured_outputs is not None
1257
+ assert out.structured_outputs.structural_tag is not None
vllm/entrypoints/openai/api_server.py
+15
@@ -321,6 +321,21 @@ async def init_app_state(
321
supported_tasks: tuple["SupportedTask", ...] | None = None,
322
) -> None:
323
vllm_config = engine_client.vllm_config
324
+
325
+ # Propagate enable_in_reasoning to the API-server process. The engine core
326
+ # runs in a separate process, so the contextvar that backs
327
+ # `get_current_vllm_config_or_none()` is None on this stack. Tool parsers
328
+ # call `get_enable_structured_outputs_in_reasoning()` during request
329
+ # handling and need to see the real flag, otherwise they silently fall
330
+ # back to False and mismatch the engine-side bitmask gating.
331
+ from vllm.tool_parsers.structural_tag_registry import (
332
+ set_enable_structured_outputs_in_reasoning,
333
+ )
334
+
335
+ set_enable_structured_outputs_in_reasoning(
336
+ vllm_config.structured_outputs_config.enable_in_reasoning
337
+ )
338
+
339
if supported_tasks is None:
340
warnings.warn(
341
"The 'supported_tasks' parameter was not provided to "
vllm/envs.py
+7
@@ -226,6 +226,7 @@ if TYPE_CHECKING:
226
VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS: bool = False
227
VLLM_SYSTEM_START_DATE: str | None = None
228
VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY: bool = False
229
+ VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = False
230
VLLM_CUSTOM_SCOPES_FOR_PROFILING: bool = False
231
VLLM_NVTX_SCOPES_FOR_PROFILING: bool = False
232
VLLM_KV_EVENTS_USE_INT_BLOCK_HASHES: bool = True
@@ -1593,6 +1594,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
1594
"VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY": lambda: bool(
1595
int(os.getenv("VLLM_TOOL_JSON_ERROR_AUTOMATIC_RETRY", "0"))
1596
),
1597
+ # When 1,the model structural tags will be used to enforce the model
1598
+ # output conforming to the model's tool-calling format and schema.
1599
+ # Default 0 (off).
1600
+ "VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: bool(
1601
+ int(os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "0"))
1602
+ ),
1603
# Add optional custom scopes for profiling, disable to avoid overheads
1604
"VLLM_CUSTOM_SCOPES_FOR_PROFILING": lambda: bool(
1605
int(os.getenv("VLLM_CUSTOM_SCOPES_FOR_PROFILING", "0"))
vllm/tool_parsers/abstract_tool_parser.py
+36
-4
@@ -2,6 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import importlib
5
+import json
6
import os
7
from collections.abc import Callable, Sequence
8
from functools import cached_property
@@ -13,6 +14,7 @@ from openai.types.responses import (
14
from openai.types.responses.function_tool import FunctionTool
15
16
from vllm.entrypoints.openai.chat_completion.protocol import (
17
+ ChatCompletionNamedToolChoiceParam,
18
ChatCompletionRequest,
19
ChatCompletionToolsParam,
20
)
@@ -23,6 +25,7 @@ from vllm.entrypoints.openai.engine.protocol import (
25
from vllm.entrypoints.openai.responses.protocol import (
26
ResponsesRequest,
27
)
28
+from vllm.envs import VLLM_ENFORCE_STRICT_TOOL_CALLING
29
from vllm.logger import init_logger
30
from vllm.sampling_params import (
31
StructuredOutputsParams,
@@ -83,13 +86,39 @@ class ToolParser:
86
return self.model_tokenizer.get_vocab()
87
88
def adjust_request(
86
- self, request: ChatCompletionRequest | ResponsesRequest
89
+ self,
90
+ request: ChatCompletionRequest | ResponsesRequest,
91
) -> ChatCompletionRequest | ResponsesRequest:
88
- """
89
- Static method that used to adjust the request parameters.
90
- """
92
+ # If there are no tools, return the request as is.
93
if not request.tools:
94
return request
95
+
96
+ # Step 1 (highest priority for ChatCompletionRequest): apply
97
+ # vLLM-owned structural tag support for model-specific tool formats.
98
+ if (
99
+ isinstance(request, ChatCompletionRequest)
100
+ and VLLM_ENFORCE_STRICT_TOOL_CALLING
101
+ ):
102
+ need_tool_calling = (
103
+ request.tool_choice == "auto"
104
+ or request.tool_choice == "required"
105
+ or isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
106
+ )
107
+ if need_tool_calling:
108
+ structure_tag = self.get_structural_tag(request)
109
+ if structure_tag is not None:
110
+ if request.structured_outputs is None:
111
+ request.structured_outputs = StructuredOutputsParams(
112
+ structural_tag=json.dumps(structure_tag.model_dump()),
113
+ )
114
+ else:
115
+ request.structured_outputs.structural_tag = json.dumps(
116
+ structure_tag.model_dump()
117
+ )
118
+ return request
119
+
120
+ # Step 2: set structured output params when tool constraints are
121
+ # derived from the tool schema.
122
json_schema_from_tool = get_json_schema_from_tools(
123
tool_choice=request.tool_choice, tools=request.tools
124
)
@@ -121,6 +150,9 @@ class ToolParser:
150
151
return request
152
153
+ def get_structural_tag(self, request: ChatCompletionRequest):
154
+ return None
155
+
156
def extract_tool_calls(
157
self, model_output: str, request: ChatCompletionRequest
158
) -> ExtractedToolCallInformation:
vllm/tool_parsers/deepseekv4_tool_parser.py
+15
@@ -1,7 +1,14 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+from vllm.entrypoints.openai.chat_completion.protocol import (
5
+ ChatCompletionRequest,
6
+)
7
from vllm.tool_parsers.deepseekv32_tool_parser import DeepSeekV32ToolParser
8
+from vllm.tool_parsers.structural_tag_registry import (
9
+ get_enable_structured_outputs_in_reasoning,
10
+ get_model_structural_tag,
11
+)
12
13
14
class DeepSeekV4ToolParser(DeepSeekV32ToolParser):
@@ -14,3 +21,11 @@ class DeepSeekV4ToolParser(DeepSeekV32ToolParser):
21
22
tool_call_start_token: str = "<｜DSML｜tool_calls>"
23
tool_call_end_token: str = "</｜DSML｜tool_calls>"
24
+
25
+ def get_structural_tag(self, request: ChatCompletionRequest):
26
+ return get_model_structural_tag(
27
+ model="deepseek_v4",
28
+ tools=request.tools,
29
+ tool_choice=request.tool_choice,
30
+ reasoning=get_enable_structured_outputs_in_reasoning(),
31
+ )
vllm/tool_parsers/qwen3coder_tool_parser.py
+14
@@ -25,12 +25,18 @@ from vllm.tool_parsers.abstract_tool_parser import (
25
Tool,
26
ToolParser,
27
)
28
+from vllm.tool_parsers.structural_tag_registry import (
29
+ get_enable_structured_outputs_in_reasoning,
30
+ get_model_structural_tag,
31
+)
32
from vllm.tool_parsers.utils import find_tool_properties
33
34
logger = init_logger(__name__)
35
36
37
class Qwen3CoderToolParser(ToolParser):
38
+ supports_required_and_named: bool = False
39
+
40
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
41
super().__init__(tokenizer, tools)
42
@@ -681,3 +687,11 @@ class Qwen3CoderToolParser(ToolParser):
687
return result
688
689
return None
690
+
691
+ def get_structural_tag(self, request: ChatCompletionRequest):
692
+ return get_model_structural_tag(
693
+ model="qwen_3_5",
694
+ tools=request.tools,
695
+ tool_choice=request.tool_choice,
696
+ reasoning=get_enable_structured_outputs_in_reasoning(),
697
+ )
vllm/tool_parsers/structural_tag_registry.py
new
+330
@@ -0,0 +1,330 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+# Model-specific structural tag builders adapted from XGrammar's
5
+# builtin structural tag implementations:
6
+# https://github.com/mlc-ai/xgrammar/blob/main/python/xgrammar/builtin_structural_tag.py
7
+
8
+from collections.abc import Callable
9
+from typing import Any, Literal
10
+
11
+from xgrammar import StructuralTag
12
+from xgrammar.structural_tag import (
13
+ AnyTextFormat,
14
+ ConstStringFormat,
15
+ JSONSchemaFormat,
16
+ SequenceFormat,
17
+ TagFormat,
18
+ TagsWithSeparatorFormat,
19
+ TriggeredTagsFormat,
20
+)
21
+
22
+from vllm.entrypoints.openai.chat_completion.protocol import (
23
+ ChatCompletionNamedToolChoiceParam,
24
+ ChatCompletionToolsParam,
25
+)
26
+
27
+SimplifiedToolChoice = Literal["auto", "required", "forced"]
28
+ToolChoice = (
29
+ Literal["none", "auto", "required"] | ChatCompletionNamedToolChoiceParam | None
30
+)
31
+StructuralTagBuilder = Callable[
32
+ [list[ChatCompletionToolsParam], SimplifiedToolChoice, bool],
33
+ StructuralTag,
34
+]
35
+
36
+_structural_tag_registry: dict[str, StructuralTagBuilder] = {}
37
+
38
+
39
+def register_model_structural_tag(name: str):
40
+ """Register a vLLM-owned model-specific structural tag builder."""
41
+
42
+ def decorator(func: StructuralTagBuilder) -> StructuralTagBuilder:
43
+ _structural_tag_registry[name] = func
44
+ return func
45
+
46
+ return decorator
47
+
48
+
49
+def get_model_structural_tag(
50
+ model: str,
51
+ tools: list[ChatCompletionToolsParam] | None,
52
+ tool_choice: ToolChoice,
53
+ reasoning: bool,
54
+) -> StructuralTag | None:
55
+ """Build a structural tag from vLLM-owned model-specific builders."""
56
+
57
+ builder = _structural_tag_registry.get(model)
58
+ if builder is None:
59
+ supported = list(_structural_tag_registry.keys())
60
+ raise ValueError(f"Unknown format type: {model}, supported types: {supported}")
61
+
62
+ normalized_tools, simplified_tool_choice = _normalize_tool_choice(
63
+ tools=tools,
64
+ tool_choice=tool_choice,
65
+ )
66
+ if not normalized_tools:
67
+ return None
68
+
69
+ return builder(normalized_tools, simplified_tool_choice, reasoning)
70
+
71
+
72
+def _normalize_tool_choice(
73
+ tools: list[ChatCompletionToolsParam] | None,
74
+ tool_choice: ToolChoice,
75
+) -> tuple[list[ChatCompletionToolsParam], SimplifiedToolChoice]:
76
+ """Normalize vLLM ChatCompletion tool_choice for structural tag builders."""
77
+
78
+ if not tools:
79
+ return [], "auto"
80
+
81
+ if tool_choice is None or tool_choice == "none":
82
+ return [], "auto"
83
+
84
+ if tool_choice == "auto":
85
+ return tools, "auto"
86
+
87
+ if tool_choice == "required":
88
+ return tools, "required"
89
+
90
+ if isinstance(tool_choice, ChatCompletionNamedToolChoiceParam):
91
+ tool_name = tool_choice.function.name
92
+ filtered_tools = [tool for tool in tools if tool.function.name == tool_name]
93
+ if not filtered_tools:
94
+ raise ValueError(
95
+ f"The tool with name '{tool_name}' is not found in the tools list."
96
+ )
97
+ return filtered_tools, "forced"
98
+
99
+ raise ValueError(f"Unsupported tool_choice for structural tag: {tool_choice}")
100
+
101
+
102
+def _get_function_parameters(function: Any) -> dict[str, Any] | bool:
103
+ """Return the JSON schema used for constrained tool arguments."""
104
+
105
+ if getattr(function, "strict", None) is False:
106
+ return True
107
+ if function.parameters is None:
108
+ return True
109
+ return function.parameters
110
+
111
+
112
+_enable_structured_outputs_in_reasoning: bool = False
113
+
114
+
115
+def set_enable_structured_outputs_in_reasoning(enabled: bool) -> None:
116
+ """Publish the engine's ``enable_in_reasoning`` flag to tool parsers.
117
+
118
+ Called once during APIServer startup so request-time parsers can read
119
+ it without going through the EngineCore-only contextvar.
120
+ """
121
+
122
+ global _enable_structured_outputs_in_reasoning
123
+ _enable_structured_outputs_in_reasoning = bool(enabled)
124
+
125
+
126
+def get_enable_structured_outputs_in_reasoning() -> bool:
127
+ """Whether structured outputs are active during the reasoning phase.
128
+
129
+ When ``True``, the structural tag will cover the reasoning part:
130
+ ``<think>...</think>`` prefix (if available); when ``False`` (default), the tag only
131
+ constrains the post-reasoning suffix.
132
+ """
133
+
134
+ return _enable_structured_outputs_in_reasoning
135
+
136
+
137
+@register_model_structural_tag("deepseek_v4")
138
+def get_deepseek_v4_structural_tag(
139
+ tools: list[ChatCompletionToolsParam],
140
+ tool_choice: SimplifiedToolChoice,
141
+ reasoning: bool,
142
+) -> StructuralTag:
143
+ """Build DeepSeek V4 structural tags."""
144
+
145
+ invoke_begin_prefix = '<｜DSML｜invoke name="'
146
+ invoke_begin_suffix = '">\n'
147
+ invoke_end = "</｜DSML｜invoke>\n"
148
+ tool_calls_prefix = "\n\n"
149
+ function_calls_begin = "<｜DSML｜tool_calls>\n"
150
+ function_calls_end = "</｜DSML｜tool_calls>"
151
+ function_calls_trigger = "<｜DSML｜tool_calls>"
152
+ think_tag_end = "</think>"
153
+ think_exclude_tokens = ["<think>", "</think>"]
154
+ xml_style = "deepseek_xml"
155
+
156
+ if tool_choice == "auto":
157
+ tags = []
158
+ for tool in tools:
159
+ function = tool.function
160
+ parameters = _get_function_parameters(function)
161
+ tags.append(
162
+ TagFormat(
163
+ begin=invoke_begin_prefix + function.name + invoke_begin_suffix,
164
+ content=JSONSchemaFormat(
165
+ json_schema=parameters,
166
+ style=xml_style,
167
+ ),
168
+ end=invoke_end,
169
+ )
170
+ )
171
+
172
+ if tags:
173
+ function_calling_tags = TagsWithSeparatorFormat(
174
+ tags=tags,
175
+ separator="\n",
176
+ at_least_one=True,
177
+ )
178
+ suffix_tag = TriggeredTagsFormat(
179
+ triggers=[function_calls_trigger],
180
+ tags=[
181
+ TagFormat(
182
+ begin=function_calls_begin,
183
+ content=function_calling_tags,
184
+ end=function_calls_end,
185
+ )
186
+ ],
187
+ excludes=think_exclude_tokens,
188
+ )
189
+ else:
190
+ suffix_tag = AnyTextFormat(excludes=think_exclude_tokens)
191
+
192
+ elif tool_choice == "forced":
193
+ if not tools:
194
+ raise ValueError("Forced tool choice must resolve to exactly one tool.")
195
+ function = tools[0].function
196
+ suffix_tag = SequenceFormat(
197
+ elements=[
198
+ ConstStringFormat(value=tool_calls_prefix + function_calls_begin),
199
+ TagFormat(
200
+ begin=invoke_begin_prefix + function.name + invoke_begin_suffix,
201
+ content=JSONSchemaFormat(
202
+ json_schema=_get_function_parameters(function),
203
+ style=xml_style,
204
+ ),
205
+ end=invoke_end,
206
+ ),
207
+ ConstStringFormat(value=function_calls_end),
208
+ ]
209
+ )
210
+
211
+ elif tool_choice == "required":
212
+ tags = []
213
+ for tool in tools:
214
+ function = tool.function
215
+ parameters = _get_function_parameters(function)
216
+ tags.append(
217
+ TagFormat(
218
+ begin=invoke_begin_prefix + function.name + invoke_begin_suffix,
219
+ content=JSONSchemaFormat(
220
+ json_schema=parameters,
221
+ style=xml_style,
222
+ ),
223
+ end=invoke_end,
224
+ )
225
+ )
226
+ assert len(tags) > 0
227
+ suffix_tag = SequenceFormat(
228
+ elements=[
229
+ ConstStringFormat(value=tool_calls_prefix + function_calls_begin),
230
+ TagsWithSeparatorFormat(
231
+ tags=tags,
232
+ separator="\n",
233
+ at_least_one=True,
234
+ ),
235
+ ConstStringFormat(value=function_calls_end),
236
+ ]
237
+ )
238
+
239
+ if not reasoning:
240
+ return StructuralTag(format=suffix_tag)
241
+
242
+ prefix_tag = TagFormat(begin="", content=AnyTextFormat(), end=think_tag_end)
243
+ return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag]))
244
+
245
+
246
+@register_model_structural_tag("qwen_3_5")
247
+def get_qwen_3_5_structural_tag(
248
+ tools: list[ChatCompletionToolsParam],
249
+ tool_choice: SimplifiedToolChoice,
250
+ reasoning: bool,
251
+) -> StructuralTag:
252
+ """Build Qwen XML structural tags.
253
+
254
+ This format is used for Qwen3-Coder/Qwen3.5/Qwen3.6 and is compatible with
255
+ Qwen variants that use the same XML tool-call format.
256
+ """
257
+ tool_call_begin_prefix = "<tool_call>\n<function="
258
+ tool_call_begin_suffix = ">\n"
259
+ tool_call_end = "\n</function>\n</tool_call>"
260
+ tool_call_trigger = "<tool_call>\n<function="
261
+ think_tag_end = "</think>"
262
+ think_suffix = "\n\n"
263
+ think_exclude_tokens = ["<think>", "</think>"]
264
+
265
+ if tool_choice == "auto":
266
+ tags = []
267
+ for tool in tools:
268
+ function = tool.function
269
+ parameters = _get_function_parameters(function)
270
+ tags.append(
271
+ TagFormat(
272
+ begin=f"{tool_call_begin_prefix}{function.name}{tool_call_begin_suffix}",
273
+ content=JSONSchemaFormat(json_schema=parameters, style="qwen_xml"),
274
+ end=tool_call_end,
275
+ )
276
+ )
277
+
278
+ if tags:
279
+ suffix_tag = TriggeredTagsFormat(
280
+ triggers=[tool_call_trigger],
281
+ tags=tags,
282
+ excludes=think_exclude_tokens,
283
+ )
284
+ else:
285
+ suffix_tag = AnyTextFormat(excludes=think_exclude_tokens)
286
+
287
+ elif tool_choice == "forced":
288
+ if not tools:
289
+ raise ValueError("Forced tool choice must resolve to exactly one tool.")
290
+ function = tools[0].function
291
+ suffix_tag = TagFormat(
292
+ begin=f"{tool_call_begin_prefix}{function.name}{tool_call_begin_suffix}",
293
+ content=JSONSchemaFormat(
294
+ json_schema=_get_function_parameters(function),
295
+ style="qwen_xml",
296
+ ),
297
+ end=tool_call_end,
298
+ )
299
+
300
+ elif tool_choice == "required":
301
+ tags = []
302
+ for tool in tools:
303
+ function = tool.function
304
+ parameters = _get_function_parameters(function)
305
+ tags.append(
306
+ TagFormat(
307
+ begin=f"{tool_call_begin_prefix}{function.name}{tool_call_begin_suffix}",
308
+ content=JSONSchemaFormat(json_schema=parameters, style="qwen_xml"),
309
+ end=tool_call_end,
310
+ )
311
+ )
312
+ assert len(tags) > 0
313
+ suffix_tag = TagsWithSeparatorFormat(
314
+ tags=tags,
315
+ separator="",
316
+ at_least_one=True,
317
+ )
318
+
319
+ if not reasoning:
320
+ result = StructuralTag(format=suffix_tag)
321
+ else:
322
+ prefix_tag = SequenceFormat(
323
+ elements=[
324
+ TagFormat(begin="", content=AnyTextFormat(), end=think_tag_end),
325
+ ConstStringFormat(value=think_suffix),
326
+ ]
327
+ )
328
+ result = StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag]))
329
+
330
+ return result