[Frontend] Upgrade XGrammar to 0.2.7 and Rust structural tags to 0.3.0 (#57272)
Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Bugen Zhao committed
Sep 18, 2026 at 19:44 UTC
c58532c86b8ab821fc1d819636480b7588bce7b5
15 files changed
+57
-303
requirements/common.txt
+1
-1
@@ -26,7 +26,7 @@ lm-format-enforcer == 0.11.3
26
llguidance >= 1.7.0, < 1.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "s390x"
27
outlines_core == 0.2.14
28
lark == 1.2.2
29
-xgrammar >= 0.2.1, < 1.0.0; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le"
29
+xgrammar == 0.2.7; platform_machine == "x86_64" or platform_machine == "aarch64" or platform_machine == "arm64" or platform_machine == "s390x" or platform_machine == "ppc64le"
30
typing_extensions >= 4.10
31
filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
32
partial-json-parser # used for parsing partial JSON outputs
requirements/test/cpu.txt
+2
-2
@@ -43,7 +43,7 @@ anyio==4.14.1
43
# sse-starlette
44
# starlette
45
# watchfiles
46
-apache-tvm-ffi==0.1.9
46
+apache-tvm-ffi==0.1.14.post0
47
# via xgrammar
48
arctic-inference==0.1.1
49
# via -r requirements/test/cuda.in
@@ -1298,7 +1298,7 @@ word2number==1.1
1298
# via lm-eval
1299
wrapt==1.17.2
1300
# via smart-open
1301
-xgrammar==0.2.3
1301
+xgrammar==0.2.7
1302
# via -r requirements/test/../common.txt
1303
xxhash==3.5.0
1304
# via
requirements/test/cuda.txt
+1
-1
@@ -1409,7 +1409,7 @@ word2number==1.1
1409
# via lm-eval
1410
wrapt==1.17.2
1411
# via smart-open
1412
-xgrammar==0.2.3
1412
+xgrammar==0.2.7
1413
# via
1414
# -c requirements/common.txt
1415
# -r requirements/test/../common.txt
requirements/test/rocm.txt
+1
-1
@@ -1360,7 +1360,7 @@ word2number==1.1
1360
# via lm-eval
1361
wrapt==2.1.2
1362
# via smart-open
1363
-xgrammar==0.2.1
1363
+xgrammar==0.2.7
1364
# via
1365
# -c requirements/common.txt
1366
# -r requirements/test/../common.txt
requirements/test/xpu.txt
+1
-1
@@ -1197,7 +1197,7 @@ werkzeug==3.1.8
1197
# via schemathesis
1198
word2number==1.1
1199
# via lm-eval
1200
-xgrammar==0.2.3
1200
+xgrammar==0.2.7
1201
# via
1202
# -c requirements/common.txt
1203
# -r requirements/test/../common.txt
rust/Cargo.lock
+2
-2
@@ -6596,9 +6596,9 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
6596
6597
[[package]]
6598
name = "xgrammar-structural-tag"
6599
-version = "0.2.0+xgrammar.0.2.4.dd729e7"
6599
+version = "0.3.0+xgrammar.0.2.7.82505d0"
6600
source = "registry+https://github.com/rust-lang/crates.io-index"
6601
-checksum = "d4d24c842efc3c24e9756aa426d530cbdac0980e49af223cb384e276e981ca0a"
6601
+checksum = "3864005575c2ab76057f789898f59fb9fa1378d07c5ceabc24764e012d0e90bb"
6602
dependencies = [
6603
"auto_impl",
6604
"serde",
rust/Cargo.toml
+1
-1
@@ -153,7 +153,7 @@ vllm-text = { path = "src/text" }
153
vllm-tokenizer = { path = "src/tokenizer" }
154
vllm-tracing = { path = "src/tracing" }
155
winnow = { version = "1.0.2", features = ["simd"] }
156
-xgrammar-structural-tag = "0.2.0"
156
+xgrammar-structural-tag = "0.3.0"
157
zeromq = { version = "0.6.0", default-features = false, features = [
158
"tokio-runtime",
159
"all-transport",
rust/src/parser/src/tool/deepseek_dsml/deepseek_v41.rs
+1
-3
@@ -4,8 +4,6 @@
4
use super::{DeepSeekDsmlToolParser, DsmlTokens};
5
use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutput};
6
7
-mod structural_tag;
8
-
7
/// Tool parser for DeepSeek V4.1's spaced DSML tags.
8
///
9
/// Arguments are emitted only after a full `invoke` block is parsed.
@@ -27,7 +25,7 @@ impl ToolParser for DeepSeekV41ToolParser {
25
}
26
27
fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
30
- Some(&structural_tag::DeepSeekV41StructuralTagBuilder)
28
+ Some(xgrammar_structural_tag::Model::DeepSeekV41.builder())
29
}
30
31
fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/structural_tag.rs
deleted
-108
@@ -1,108 +0,0 @@
1
-// SPDX-License-Identifier: Apache-2.0
2
-// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-//! Visible-text grammar for DeepSeek V4.1 tool calls.
5
-//!
6
-//! Adapted from xgrammar-structural-tag 0.2.0's DeepSeek V4 builder, with
7
-//! V4.1's spaced DSML tags and schema-unconstrained parameter values.
8
-
9
-use serde_json::Value;
10
-use xgrammar_structural_tag::Result;
11
-use xgrammar_structural_tag::builders::{
12
- StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
13
-};
14
-use xgrammar_structural_tag::format::{
15
- Format, JsonSchemaFormat, StructuralTag, TagFormat, TriggeredTagsFormat,
16
-};
17
-use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam};
18
-
19
-use super::DsmlTokens;
20
-
21
-const TOKENS: DsmlTokens = DsmlTokens::V41;
22
-
23
-pub(super) struct DeepSeekV41StructuralTagBuilder;
24
-
25
-impl StructuralTagBuilder for DeepSeekV41StructuralTagBuilder {
26
- fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
27
- let tags = ctx
28
- .function_tools
29
- .iter()
30
- .map(|tool| call_tag(tool, ctx.options))
31
- .collect::<Vec<_>>();
32
- let excludes = if ctx.options.exclude_special_tokens {
33
- &["<think>", "</think>"][..]
34
- } else {
35
- &[]
36
- };
37
- let format = match ctx.tool_choice {
38
- BuilderToolChoice::Auto if tags.is_empty() => Format::any_text_excluding(excludes),
39
- BuilderToolChoice::Auto => Format::TriggeredTags(
40
- TriggeredTagsFormat::new(
41
- &[TOKENS.tool_calls_start],
42
- vec![TagFormat::new(
43
- format!("{}\n", TOKENS.tool_calls_start),
44
- Format::tags_with_separator(tags, "", true, false),
45
- TOKENS.tool_calls_end,
46
- )],
47
- )
48
- .with_excludes(excludes),
49
- ),
50
- BuilderToolChoice::Forced | BuilderToolChoice::Required => Format::sequence(vec![
51
- Format::const_string(format!("\n\n{}\n", TOKENS.tool_calls_start)),
52
- Format::tags_with_separator(
53
- tags,
54
- "",
55
- true,
56
- ctx.tool_choice == BuilderToolChoice::Forced,
57
- ),
58
- Format::const_string(TOKENS.tool_calls_end),
59
- ]),
60
- };
61
- // Reasoning boundaries remain owned by the engine reasoner; serving
62
- // calls this visible-text builder with reasoning=false.
63
- Ok(StructuralTag::new(format))
64
- }
65
-}
66
-
67
-fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat {
68
- // TODO: enforce tool.parameters and strict schema semantics once V4.1
69
- // argument conversion is available. This builder intentionally ignores
70
- // parameters and strict: it constrains tool names and DSML/value syntax
71
- // only. Parameter names, presence, uniqueness and value schemas remain
72
- // unconstrained, including for strict=true. Serving still uses strict to
73
- // decide whether auto tool choice activates a grammar.
74
- TagFormat::new(
75
- format!("{} name=\"{}\">\n", TOKENS.invoke_start, tool.function.name),
76
- Format::star(parameter(options)),
77
- format!("{}\n", TOKENS.invoke_end),
78
- )
79
-}
80
-
81
-fn parameter(options: StructuralTagOptions) -> Format {
82
- Format::tag(
83
- format!("{} name=\"", TOKENS.parameter_start),
84
- Format::sequence(vec![
85
- Format::regex(r#"[^"]+"#),
86
- Format::const_string("\" string=\""),
87
- Format::or(vec![
88
- Format::sequence(vec![
89
- Format::const_string("true\">"),
90
- Format::any_text_excluding(&[
91
- TOKENS.parameter_end,
92
- TOKENS.invoke_end,
93
- TOKENS.tool_calls_end,
94
- ]),
95
- ]),
96
- Format::sequence(vec![
97
- Format::const_string("false\">"),
98
- Format::JsonSchema(
99
- JsonSchemaFormat::new(Value::Bool(true))
100
- .with_any_order(options.any_order)
101
- .with_max_whitespace_cnt(options.max_whitespace_cnt),
102
- ),
103
- ]),
104
- ]),
105
- ]),
106
- format!("{}\n", TOKENS.parameter_end),
107
- )
108
-}
rust/src/parser/src/tool/deepseek_dsml/deepseek_v41/tests.rs
+8
-2
@@ -102,7 +102,7 @@ fn v4_tags_remain_text_in_v41_dialect() {
102
}
103
104
#[test]
105
-fn structural_tag_ignores_parameter_schemas_and_strict() {
105
+fn structural_tag_respects_parameter_schemas_and_strict() {
106
let parser = DeepSeekV41ToolParser::create(&tools()).unwrap();
107
let builder = parser.structural_tag_builder().unwrap();
108
let options = StructuralTagOptions::default().with_reasoning(false);
@@ -128,7 +128,13 @@ fn structural_tag_ignores_parameter_schemas_and_strict() {
128
] {
129
let mut function = FunctionDefinition::new("lookup").with_parameters(parameters);
130
function.strict = strict;
131
- assert_eq!(build(function), expected);
131
+ let tag = build(function);
132
+ if strict == Some(false) {
133
+ assert_eq!(tag, expected);
134
+ } else {
135
+ assert_ne!(tag, expected);
136
+ }
137
+ assert!(tag.to_json_string().unwrap().contains("deepseek_v4_1_xml"));
138
}
139
}
140
}
rust/src/parser/src/tool/hy/structural_tag.rs
+6
-6
@@ -219,7 +219,7 @@ mod tests {
219
ToolChoice::required(),
220
);
221
222
- expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>\n"},{"type":"tags_with_separator","tags":[{"begin":"<tool_call:opensource>get_weather<tool_sep:opensource>\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"days"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]}}]},"end":"</tool_call:opensource>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
222
+ expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>\n"},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<tool_call:opensource>get_weather<tool_sep:opensource>\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"days"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]}}]},"end":"</tool_call:opensource>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
223
}
224
225
#[test]
@@ -238,7 +238,7 @@ mod tests {
238
ToolChoice::required(),
239
);
240
241
- expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>"},{"type":"tags_with_separator","tags":[{"begin":"<tool_call:opensource>get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"}]}]},"end":"</tool_call:opensource>"}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
241
+ expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>"},{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<tool_call:opensource>get_weather","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value:opensource>"}]}]},"end":"</tool_call:opensource>"}],"separator":"","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
242
}
243
244
#[test]
@@ -259,7 +259,7 @@ mod tests {
259
);
260
let value = serde_json::to_value(tag).unwrap();
261
262
- expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"limit"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq(
262
+ expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"limit"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq(
263
&serde_json::to_string(
264
&value["format"]["elements"][1]["tags"][0]["content"],
265
)
@@ -276,8 +276,8 @@ mod tests {
276
let auto = build("", &tools, ToolChoice::auto()).to_json_string().unwrap();
277
let forced = build("", &tools, ToolChoice::function("lookup")).to_json_string().unwrap();
278
279
- expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":["<tool_calls>"],"tags":[{"begin":"<tool_calls>\n","content":{"type":"tags_with_separator","tags":[{"begin":"<tool_call>search<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"},{"begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n</tool_calls>"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto);
280
- expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls>\n"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"},{"type":"const_string","value":"\n</tool_calls>"}]}}"#]].assert_eq(&forced);
279
+ expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":["<tool_calls>"],"tags":[{"type":"tag","begin":"<tool_calls>\n","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<tool_call>search<tool_sep>\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":"</tool_call>"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":"</tool_call>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n</tool_calls>"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto);
280
+ expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls>\n"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[],"max_tokens":null,"max_chars":null},"end":"</tool_call>"},{"type":"const_string","value":"\n</tool_calls>"}]}}"#]].assert_eq(&forced);
281
}
282
283
#[test]
@@ -314,7 +314,7 @@ mod tests {
314
);
315
let value = serde_json::to_value(tag).unwrap();
316
317
- expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"tenant"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq(
317
+ expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"tenant"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"],"max_tokens":null,"max_chars":null},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq(
318
&serde_json::to_string(
319
&value["format"]["elements"][1]["tags"][0]["content"],
320
)
rust/src/parser/src/unified/kimi_k3/structural_tag.rs
+17
-8
@@ -6,14 +6,14 @@
6
use serde_json::{Map, Value};
7
use xgrammar_structural_tag::Result;
8
use xgrammar_structural_tag::builders::{
9
- StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
9
+ ReasoningMode, StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
10
};
11
use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, StructuralTag, TagFormat};
12
use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam, function_parameters};
13
14
use super::{
15
ARG_CLOSE, CALL_CLOSE, CLOSE, END_OF_MSG, JSON_CLOSE, JSON_OPEN, MESSAGE_CLOSE, OPEN,
16
- RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, TOOLS_CLOSE, TOOLS_OPEN,
16
+ RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE, THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN,
17
};
18
19
pub(super) static KIMI_K3_STRUCTURAL_TAG_BUILDER: KimiK3StructuralTagBuilder =
@@ -50,14 +50,23 @@ impl StructuralTagBuilder for KimiK3StructuralTagBuilder {
50
}
51
}
52
53
-fn response_prefix(reasoning: bool) -> Vec<Format> {
53
+fn response_prefix(reasoning: ReasoningMode) -> Vec<Format> {
54
let mut elements = Vec::new();
55
- if reasoning {
56
- elements.push(Format::tag(
57
- "",
55
+ if reasoning != ReasoningMode::Disabled {
56
+ let prefix = Format::tag(
57
+ if reasoning == ReasoningMode::Auto {
58
+ THINK_OPEN
59
+ } else {
60
+ ""
61
+ },
62
Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG]),
63
THINK_CLOSE,
60
- ));
64
+ );
65
+ elements.push(if reasoning == ReasoningMode::Auto {
66
+ Format::optional(prefix)
67
+ } else {
68
+ prefix
69
+ });
70
elements.push(Format::const_string(RESPONSE_OPEN));
71
} else {
72
elements.push(Format::optional(Format::const_string(RESPONSE_OPEN)));
@@ -386,7 +395,7 @@ mod tests {
395
)
396
.unwrap();
397
389
- expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"]},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap());
398
+ expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|open|>","<|close|>","<|end_of_msg|>"],"max_tokens":null,"max_chars":null},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"type":"tag","begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap());
399
}
400
401
#[test]
tests/tool_parsers/test_structural_tag_registry.py
+13
-58
@@ -6,7 +6,7 @@ from types import SimpleNamespace
6
from unittest.mock import MagicMock
7
8
import pytest
9
-from xgrammar import Grammar, StructuralTag, builtin_structural_tag
9
+from xgrammar import Grammar, StructuralTag
10
from xgrammar.testing import _is_grammar_accept_string
11
12
from vllm.entrypoints.openai.chat_completion.protocol import (
@@ -23,6 +23,7 @@ from vllm.tool_parsers.deepseekv31_tool_parser import DeepSeekV31ToolParser
23
from vllm.tool_parsers.deepseekv32_engine_tool_parser import (
24
DeepSeekV32EngineToolParser,
25
)
26
+from vllm.tool_parsers.deepseekv41_engine_tool_parser import DeepSeekV41EngineToolParser
27
from vllm.tool_parsers.glm47_moe_tool_parser import Glm47MoeModelToolParser
28
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
29
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
@@ -83,51 +84,9 @@ def test_supported_structural_tag_models_include_vllm_builtins():
84
assert "hermes" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
85
86
86
-@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"])
87
-def test_deepseek_v41_older_xgrammar_keeps_format_constraints(
88
- choice,
89
- sample_tools_strict,
90
- monkeypatch,
91
-):
92
- monkeypatch.setattr(
93
- builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None, raising=False
94
- )
95
- tool_choice = (
96
- ChatCompletionNamedToolChoiceParam(
97
- function=ChatCompletionNamedFunction(name=choice)
98
- )
99
- if choice == "get_weather"
100
- else choice
101
- )
102
- tag = get_model_structural_tag(
103
- "deepseek_v41", sample_tools_strict, tool_choice, reasoning=False
104
- )
105
- grammar = Grammar.from_structural_tag(tag)
106
- begin = '\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name="get_weather">\n'
107
- end = "</｜DSML｜ invoke>\n</｜DSML｜ calls>"
108
- parameter = (
109
- '<｜DSML｜ parameter name="undeclared" string="false">'
110
- '{"nested": [true, null, 1.5]}</｜DSML｜ parameter>\n'
111
- )
112
- # Required city is omitted; extra and repeated fields remain unconstrained.
113
- assert _is_grammar_accept_string(grammar, begin + end)
114
- assert _is_grammar_accept_string(grammar, begin + parameter * 2 + end)
115
- assert not _is_grammar_accept_string(
116
- grammar,
117
- begin + parameter.replace('{"nested": [true, null, 1.5]}', "invalid") + end,
118
- )
119
- assert not _is_grammar_accept_string(
120
- grammar, (begin + end).replace('name="get_weather"', 'name="unknown"')
121
- )
122
- assert not _is_grammar_accept_string(
123
- grammar, begin + parameter.replace("｜ parameter", "｜parameter") + end
124
- )
125
- assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto")
126
-
127
-
87
def test_deepseek_v41_named_choice_emits_one_call(sample_tools):
88
tag = get_model_structural_tag(
130
- "deepseek_v41",
89
+ "deepseek_v4_1",
90
sample_tools,
91
ChatCompletionNamedToolChoiceParam(
92
function=ChatCompletionNamedFunction(name="get_weather")
@@ -147,19 +106,14 @@ def test_deepseek_v41_named_choice_emits_one_call(sample_tools):
106
grammar, "\n\n<｜DSML｜ calls>\n" + call * 2 + "</｜DSML｜ calls>"
107
)
108
assert (
150
- get_model_structural_tag("deepseek_v41", sample_tools, "auto", reasoning=False)
109
+ get_model_structural_tag("deepseek_v4_1", sample_tools, "auto", reasoning=False)
110
is None
111
)
112
113
155
-@pytest.mark.skipif(
156
- not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"),
157
- reason="Requires XGrammar's DeepSeek V4.1 builtin",
158
-)
114
@pytest.mark.parametrize("choice", ["auto", "required", "get_weather"])
160
-@pytest.mark.parametrize("reasoning", [False, True])
115
def test_deepseek_v41_constrains_parameters_after_reasoning(
162
- sample_tools_strict, choice, reasoning
116
+ sample_tools_strict, choice
117
):
118
"""Strict calls must enforce the schema without requiring another think close."""
119
tool_choice = (
@@ -171,7 +125,7 @@ def test_deepseek_v41_constrains_parameters_after_reasoning(
125
)
126
sample_tools_strict[0].function.parameters["additionalProperties"] = False
127
tag = get_model_structural_tag(
174
- "deepseek_v41", sample_tools_strict, tool_choice, reasoning=reasoning
128
+ "deepseek_v4_1", sample_tools_strict, tool_choice, reasoning=False
129
)
130
grammar = Grammar.from_structural_tag(tag)
131
begin = '\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name="get_weather">\n'
@@ -195,14 +149,10 @@ def test_deepseek_v41_constrains_parameters_after_reasoning(
149
assert _is_grammar_accept_string(grammar, "Hello") == (choice == "auto")
150
151
198
-@pytest.mark.skipif(
199
- not hasattr(builtin_structural_tag, "get_deepseek_v4_1_structural_tag"),
200
- reason="Requires XGrammar's DeepSeek V4.1 builtin",
201
-)
152
def test_deepseek_v41_non_strict_parallel_calls_keep_typed_dsml(sample_tools):
153
sample_tools[0].function.strict = False
154
tag = get_model_structural_tag(
205
- "deepseek_v41", sample_tools, "required", reasoning=False
155
+ "deepseek_v4_1", sample_tools, "required", reasoning=False
156
)
157
grammar = Grammar.from_structural_tag(tag)
158
call = (
@@ -571,6 +521,7 @@ def test_get_model_structural_tag_supports_named_tool_choice(
521
(DeepSeekV31ToolParser, "deepseek_v3_1"),
522
(DeepSeekV32EngineToolParser, "deepseek_v3_2"),
523
(DeepSeekV4EngineToolParser, "deepseek_v4"),
524
+ (DeepSeekV41EngineToolParser, "deepseek_v4_1"),
525
(Glm47MoeModelToolParser, "glm_4_7"),
526
(Hermes2ProToolParser, "hermes"),
527
(KimiK2ToolParser, "kimi"),
@@ -638,7 +589,11 @@ def test_get_structural_tag_disables_reasoning(
589
assert captured == [False]
590
591
592
+@pytest.mark.parametrize(
593
+ "parser_cls", [Qwen3EngineToolParser, DeepSeekV41EngineToolParser]
594
+)
595
def test_unified_parser_get_structural_tag_disables_reasoning(
596
+ parser_cls,
597
monkeypatch: pytest.MonkeyPatch,
598
sample_tools_strict: list[ChatCompletionToolsParam],
599
):
@@ -654,7 +609,7 @@ def test_unified_parser_get_structural_tag_disables_reasoning(
609
)
610
611
class TestParser(DelegatingParser):
657
- tool_parser_cls = Qwen3EngineToolParser
612
+ tool_parser_cls = parser_cls
613
614
request = ChatCompletionRequest(
615
messages=[],
vllm/tool_parsers/deepseekv41_engine_tool_parser.py
+1
-1
@@ -5,4 +5,4 @@ from vllm.parser.engine.registered_adapters import DeepSeekV41ParserToolAdapter
5
6
7
class DeepSeekV41EngineToolParser(DeepSeekV41ParserToolAdapter): # type: ignore[valid-type, misc]
8
- structural_tag_model = "deepseek_v41"
8
+ structural_tag_model = "deepseek_v4_1"
vllm/tool_parsers/structural_tag_registry.py
+2
-108
@@ -10,7 +10,6 @@ from openai.types.responses.tool import Tool as ResponsesTool
10
from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed
11
from openai.types.responses.tool_choice_function import ToolChoiceFunction
12
from xgrammar import StructuralTag, normalize_tool_choice
13
-from xgrammar import builtin_structural_tag as xgrammar_builtin_structural_tag
13
from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag
14
from xgrammar.openai_tool_call_schema import (
15
BuiltinToolParam,
@@ -71,11 +70,10 @@ XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
70
"deepseek_v3_2",
71
"glm_4_7",
72
"deepseek_v4",
73
+ "deepseek_v4_1",
74
}
75
)
76
-VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
77
- {"deepseek_v41", "hermes", "hy_v4", "kimi_k3"}
78
-)
76
+VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset({"hermes", "hy_v4", "kimi_k3"})
77
SUPPORTED_STRUCTURAL_TAG_MODELS = (
78
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
79
)
@@ -270,110 +268,6 @@ def get_function_parameters(function) -> dict[str, Any] | bool:
268
return function.parameters if function.parameters is not None else True
269
270
273
-_V41_CALLS_START = "<｜DSML｜ calls>"
274
-_V41_CALLS_END = "</｜DSML｜ calls>"
275
-_V41_INVOKE_END = "</｜DSML｜ invoke>"
276
-_V41_PARAMETER_END = "</｜DSML｜ parameter>"
277
-
278
-
279
-@register_vllm_structural_tag("deepseek_v41")
280
-def get_deepseek_v41_structural_tag(
281
- tools: list[FunctionToolParam],
282
- builtin_tools: list[BuiltinToolParam],
283
- tool_choice: SimplifiedToolChoice,
284
- reasoning: bool,
285
- token_suffix: str = "",
286
-) -> StructuralTag:
287
- # Serving enables this visible-text grammar after the reasoning boundary.
288
- builder = getattr(
289
- xgrammar_builtin_structural_tag, "get_deepseek_v4_1_structural_tag", None
290
- )
291
- if builder is not None:
292
- return builder(
293
- tools=tools,
294
- builtin_tools=builtin_tools,
295
- tool_choice=tool_choice,
296
- reasoning="disabled",
297
- )
298
-
299
- del builtin_tools, reasoning, token_suffix
300
-
301
- # Compatibility with xgrammar releases without the V4.1 builtin. This constrains
302
- # tool names and DSML/value syntax only. Parameter names, presence, uniqueness
303
- # and value schemas remain unconstrained, including for strict=true. The request
304
- # layer still uses strict to decide whether auto tool choice activates a grammar.
305
- parameter = TagFormat(
306
- begin='<｜DSML｜ parameter name="',
307
- content=SequenceFormat(
308
- elements=[
309
- RegexFormat(pattern=r'[^"]+'),
310
- ConstStringFormat(value='" string="'),
311
- OrFormat(
312
- elements=[
313
- SequenceFormat(
314
- elements=[
315
- ConstStringFormat(value='true">'),
316
- AnyTextFormat(
317
- excludes=[
318
- _V41_PARAMETER_END,
319
- _V41_INVOKE_END,
320
- _V41_CALLS_END,
321
- ]
322
- ),
323
- ]
324
- ),
325
- SequenceFormat(
326
- elements=[
327
- ConstStringFormat(value='false">'),
328
- JSONSchemaFormat(json_schema=True),
329
- ]
330
- ),
331
- ]
332
- ),
333
- ]
334
- ),
335
- end=f"{_V41_PARAMETER_END}\n",
336
- )
337
- calls = TagsWithSeparatorFormat(
338
- tags=[
339
- TagFormat(
340
- begin=f'<｜DSML｜ invoke name="{tool.function.name}">\n',
341
- content=StarFormat(content=parameter),
342
- end=f"{_V41_INVOKE_END}\n",
343
- )
344
- for tool in tools
345
- ],
346
- separator="",
347
- at_least_one=True,
348
- stop_after_first=tool_choice == "forced",
349
- )
350
- if tool_choice == "auto":
351
- return StructuralTag(
352
- format=TriggeredTagsFormat(
353
- triggers=[_V41_CALLS_START],
354
- tags=[
355
- TagFormat(
356
- begin=f"{_V41_CALLS_START}\n",
357
- content=calls,
358
- end=_V41_CALLS_END,
359
- )
360
- ],
361
- excludes=["<think>", "</think>"],
362
- )
363
- if tools
364
- else AnyTextFormat(excludes=["<think>", "</think>"])
365
- )
366
- return StructuralTag(
367
- format=SequenceFormat(
368
- elements=[
369
- ConstStringFormat(value=f"\n\n{_V41_CALLS_START}\n"),
370
- calls,
371
- ConstStringFormat(value=_V41_CALLS_END),
372
- ]
373
- )
374
- )
375
-
376
-
271
def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
272
arguments_field_prefix = '", "arguments": '
273
formats = [