@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 4ed8eaafb0

[Rust Frontend] Integrate `xgrammar-structural-tag` for `strict` and `required` tool calling (#46057)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Jun 24, 2026 at 10:46 UTC 4ed8eaafb05f8c934b180c14bf1dc450eba7b12d
23 files changed +466 -81
rust/Cargo.lock
+26
@@ -316,6 +316,17 @@ version = "1.1.2"
316 source = "registry+https://github.com/rust-lang/crates.io-index"
317 checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
318
319 +[[package]]
320 +name = "auto_impl"
321 +version = "1.3.0"
322 +source = "registry+https://github.com/rust-lang/crates.io-index"
323 +checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7"
324 +dependencies = [
325 + "proc-macro2",
326 + "quote",
327 + "syn 2.0.117",
328 +]
329 +
330 [[package]]
331 name = "autocfg"
332 version = "1.5.0"
@@ -5726,6 +5737,7 @@ dependencies = [
5737 "vllm-text",
5738 "vllm-tokenizer",
5739 "vllm-tool-parser",
5740 + "xgrammar-structural-tag",
5741 "zeromq",
5742 ]
5743
@@ -5981,6 +5993,7 @@ dependencies = [
5993 "thiserror-ext",
5994 "tool-parser",
5995 "winnow",
5996 + "xgrammar-structural-tag",
5997 ]
5998
5999 [[package]]
@@ -6582,6 +6595,19 @@ version = "0.6.2"
6595 source = "registry+https://github.com/rust-lang/crates.io-index"
6596 checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
6597
6598 +[[package]]
6599 +name = "xgrammar-structural-tag"
6600 +version = "0.1.0+xgrammar.0.2.2.4d145cc"
6601 +source = "registry+https://github.com/rust-lang/crates.io-index"
6602 +checksum = "2436dea2393d55a3b188588aa300c5a8afe8f45a77da52c611fb4498a6c876e6"
6603 +dependencies = [
6604 + "auto_impl",
6605 + "serde",
6606 + "serde_json",
6607 + "strum",
6608 + "thiserror 2.0.18",
6609 +]
6610 +
6611 [[package]]
6612 name = "y4m"
6613 version = "0.8.0"
rust/Cargo.toml
+1
@@ -123,6 +123,7 @@ vllm-text = { path = "src/text" }
123 vllm-tokenizer = { path = "src/tokenizer" }
124 vllm-tool-parser = { path = "src/tool-parser" }
125 winnow = "1.0.2"
126 +xgrammar-structural-tag = "0.1.0"
127 zeromq = { version = "0.6.0", default-features = false, features = [
128 "tokio-runtime",
129 "all-transport",
rust/src/chat/Cargo.toml
+1
@@ -34,6 +34,7 @@ vllm-reasoning-parser.workspace = true
34 vllm-text.workspace = true
35 vllm-tokenizer.workspace = true
36 vllm-tool-parser.workspace = true
37 +xgrammar-structural-tag.workspace = true
38
39 [dev-dependencies]
40 anyhow.workspace = true
rust/src/chat/src/error.rs
+2
@@ -64,6 +64,8 @@ pub enum Error {
64 StreamClosedBeforeTerminalOutput { request_id: String },
65 #[error("tool call stream state is inconsistent: {message}")]
66 ToolCallStreamInvariant { message: String },
67 + #[error("failed to build structural tag: {message}")]
68 + StructuralTag { message: String },
69 #[error(transparent)]
70 Text(#[from] vllm_text::Error),
71 #[error(transparent)]
rust/src/chat/src/output/default/mod.rs
+6 -3
@@ -1,6 +1,7 @@
1 //! Default output processing pipeline.
2
3 mod reasoning;
4 +mod structural_tag;
5 mod tool;
6
7 use std::sync::Once;
@@ -11,6 +12,7 @@ use trait_set::trait_set;
12 use vllm_text::tokenizer::DynTokenizer;
13
14 use self::reasoning::reasoning_event_stream;
15 +use self::structural_tag::apply_structural_tag_constraint;
16 use self::tool::tool_event_stream;
17 use super::structured::structured_chat_event_stream;
18 use crate::error::Result;
@@ -21,7 +23,7 @@ use crate::output::{
23 use crate::parser::ParserSelection;
24 use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};
25 use crate::parser::tool::{ToolParser, ToolParserFactory};
24 -use crate::request::{ChatRequest, ChatToolChoice};
26 +use crate::request::ChatRequest;
27 use crate::{Error, Result as ChatResult};
28
29 trait_set! {
@@ -54,8 +56,7 @@ impl DefaultChatOutputProcessor {
56 tool_call_parser: &ParserSelection,
57 reasoning_parser: &ParserSelection,
58 ) -> ChatResult<Self> {
57 - let tool_parsing_enabled =
58 - matches!(request.tool_choice, ChatToolChoice::Auto) && !request.tools.is_empty();
59 + let tool_parsing_enabled = request.tool_parsing_enabled();
60 let tool_parser = if tool_parsing_enabled {
61 Some(Self::resolve_tool_parser(
62 request,
@@ -115,6 +116,8 @@ impl DefaultChatOutputProcessor {
116 request.decode_options.skip_special_tokens = false;
117 }
118
119 + apply_structural_tag_constraint(request, parser.as_ref())?;
120 +
121 TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser"));
122 Ok(parser)
123 }
rust/src/chat/src/output/default/structural_tag.rs new
+257
@@ -0,0 +1,257 @@
1 +//! Applies xgrammar structural-tag constraints for strict tool calling.
2 +
3 +use thiserror_ext::AsReport;
4 +use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
5 +use xgrammar_structural_tag::{
6 + FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam,
7 + build_structural_tag,
8 +};
9 +
10 +use crate::parser::tool::ToolParser;
11 +use crate::request::{ChatRequest, ChatToolChoice};
12 +use crate::{Error, Result as ChatResult};
13 +
14 +/// Apply structural tag constraints to the request based on the tool parser's structural tag
15 +/// support and the request's tool choice.
16 +pub(super) fn apply_structural_tag_constraint(
17 + request: &mut ChatRequest,
18 + parser: &dyn ToolParser,
19 +) -> ChatResult<()> {
20 + let Some(model) = parser.structural_tag_model() else {
21 + return Ok(());
22 + };
23 + let Some(tool_choice) = structural_tag_tool_choice(request) else {
24 + return Ok(());
25 + };
26 +
27 + let tools = request
28 + .tools
29 + .iter()
30 + .map(|tool| {
31 + ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
32 + name: tool.name.clone(),
33 + description: tool.description.clone(),
34 + parameters: Some(tool.parameters.clone()),
35 + strict: tool.strict,
36 + }))
37 + })
38 + .collect::<Vec<_>>();
39 +
40 + let structural_tag = build_structural_tag(model, &tools, tool_choice, false)
41 + .and_then(|tag| tag.to_json_string())
42 + .map_err(|error| Error::StructuralTag {
43 + message: error.to_report_string(),
44 + })?;
45 +
46 + // Overwrite any existing structured output settings with the structural tag constraint.
47 + request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
48 + structural_tag: Some(structural_tag),
49 + backend: StructuredOutputBackend::Xgrammar,
50 + ..Default::default()
51 + });
52 +
53 + Ok(())
54 +}
55 +
56 +/// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request.
57 +///
58 +/// Returns `None` if no structural tag constraints should be applied.
59 +fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagToolChoice> {
60 + if request.tools.is_empty() {
61 + return None;
62 + }
63 +
64 + match &request.tool_choice {
65 + // For `Auto`, only apply the structural tag if there's at least one strict tool.
66 + ChatToolChoice::Auto if request.tools.iter().any(|tool| tool.strict == Some(true)) => {
67 + Some(StructuralTagToolChoice::auto())
68 + }
69 + ChatToolChoice::Auto | ChatToolChoice::None => None,
70 +
71 + ChatToolChoice::Required => Some(StructuralTagToolChoice::required()),
72 + ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())),
73 + }
74 +}
75 +
76 +#[cfg(test)]
77 +mod tests {
78 + use serde_json::{Value, json};
79 + use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams};
80 + use vllm_tool_parser::{Qwen3CoderToolParser, Tool};
81 +
82 + use super::*;
83 +
84 + fn chat_tool(name: &str, strict: Option<bool>) -> Tool {
85 + Tool {
86 + name: name.to_string(),
87 + description: None,
88 + parameters: json!({
89 + "type": "object",
90 + "properties": {
91 + "query": { "type": "string" }
92 + },
93 + "required": ["query"]
94 + }),
95 + strict,
96 + }
97 + }
98 +
99 + fn qwen3_coder_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
100 + Qwen3CoderToolParser::create(tools).expect("Qwen3 Coder parser should build")
101 + }
102 +
103 + fn request(tool_choice: ChatToolChoice, tools: Vec<Tool>) -> ChatRequest {
104 + ChatRequest {
105 + tool_choice,
106 + tools,
107 + ..ChatRequest::for_test()
108 + }
109 + }
110 +
111 + fn structural_tag_value(request: &ChatRequest) -> Value {
112 + let params = request
113 + .sampling_params
114 + .structured_outputs
115 + .as_ref()
116 + .expect("structured outputs should be set");
117 + assert_eq!(params.backend, StructuredOutputBackend::Xgrammar);
118 + serde_json::from_str(
119 + params.structural_tag.as_deref().expect("structural_tag should be set"),
120 + )
121 + .expect("structural_tag should be valid JSON")
122 + }
123 +
124 + fn structured_outputs(request: &ChatRequest) -> &StructuredOutputsParams {
125 + request
126 + .sampling_params
127 + .structured_outputs
128 + .as_ref()
129 + .expect("structured outputs should be set")
130 + }
131 +
132 + #[test]
133 + fn auto_strict_tool_choice_builds_structural_tag() {
134 + let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
135 + let parser = qwen3_coder_parser(&request.tools);
136 +
137 + apply_structural_tag_constraint(&mut request, parser.as_ref())
138 + .expect("structural tag should build");
139 +
140 + let tag = structural_tag_value(&request);
141 + assert_eq!(tag["type"], "structural_tag");
142 + assert!(tag.to_string().contains("search"));
143 + }
144 +
145 + #[test]
146 + fn auto_non_strict_tool_choice_skips_structural_tag() {
147 + let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
148 + let parser = qwen3_coder_parser(&request.tools);
149 +
150 + apply_structural_tag_constraint(&mut request, parser.as_ref())
151 + .expect("structural tag decision should succeed");
152 +
153 + assert!(request.sampling_params.structured_outputs.is_none());
154 + }
155 +
156 + #[test]
157 + fn auto_strict_tool_choice_overwrites_existing_json_guidance() {
158 + let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
159 + request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
160 + json: Some(json!({"type": "object"})),
161 + backend: StructuredOutputBackend::Xgrammar,
162 + ..Default::default()
163 + });
164 + let parser = qwen3_coder_parser(&request.tools);
165 +
166 + apply_structural_tag_constraint(&mut request, parser.as_ref())
167 + .expect("structural tag should build");
168 +
169 + let params = structured_outputs(&request);
170 + assert!(params.json.is_none());
171 + assert!(params.structural_tag.is_some());
172 + let tag = structural_tag_value(&request);
173 + assert_eq!(tag["type"], "structural_tag");
174 + assert!(tag.to_string().contains("search"));
175 + }
176 +
177 + #[test]
178 + fn required_tool_choice_builds_structural_tag_without_strict_tools() {
179 + let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
180 + let parser = qwen3_coder_parser(&request.tools);
181 +
182 + apply_structural_tag_constraint(&mut request, parser.as_ref())
183 + .expect("structural tag should build");
184 +
185 + let tag = structural_tag_value(&request);
186 + assert_eq!(tag["type"], "structural_tag");
187 + assert!(tag.to_string().contains("search"));
188 + }
189 +
190 + #[test]
191 + fn required_tool_choice_overwrites_existing_json_object_guidance() {
192 + let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
193 + request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
194 + json_object: Some(true),
195 + backend: StructuredOutputBackend::Xgrammar,
196 + ..Default::default()
197 + });
198 + let parser = qwen3_coder_parser(&request.tools);
199 +
200 + apply_structural_tag_constraint(&mut request, parser.as_ref())
201 + .expect("structural tag should build");
202 +
203 + let params = structured_outputs(&request);
204 + assert!(params.json_object.is_none());
205 + assert!(params.structural_tag.is_some());
206 + let tag = structural_tag_value(&request);
207 + assert_eq!(tag["type"], "structural_tag");
208 + assert!(tag.to_string().contains("search"));
209 + }
210 +
211 + #[test]
212 + fn named_tool_choice_builds_structural_tag_for_named_tool_only() {
213 + let mut request = request(
214 + ChatToolChoice::Function {
215 + name: "lookup".to_string(),
216 + },
217 + vec![chat_tool("search", None), chat_tool("lookup", None)],
218 + );
219 + let parser = qwen3_coder_parser(&request.tools);
220 +
221 + apply_structural_tag_constraint(&mut request, parser.as_ref())
222 + .expect("structural tag should build");
223 +
224 + let tag = structural_tag_value(&request).to_string();
225 + assert!(tag.contains("lookup"));
226 + assert!(!tag.contains("search"));
227 + }
228 +
229 + #[test]
230 + fn none_tool_choice_skips_structural_tag() {
231 + let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
232 + let parser = qwen3_coder_parser(&request.tools);
233 +
234 + apply_structural_tag_constraint(&mut request, parser.as_ref())
235 + .expect("structural tag decision should succeed");
236 +
237 + assert!(request.sampling_params.structured_outputs.is_none());
238 + }
239 +
240 + #[test]
241 + fn none_tool_choice_preserves_existing_json_object_guidance() {
242 + let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
243 + request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
244 + json_object: Some(true),
245 + backend: StructuredOutputBackend::Xgrammar,
246 + ..Default::default()
247 + });
248 + let parser = qwen3_coder_parser(&request.tools);
249 +
250 + apply_structural_tag_constraint(&mut request, parser.as_ref())
251 + .expect("structural tag decision should succeed");
252 +
253 + let params = structured_outputs(&request);
254 + assert_eq!(params.json_object, Some(true));
255 + assert!(params.structural_tag.is_none());
256 + }
257 +}
rust/src/chat/src/request.rs
+7 -3
@@ -382,12 +382,16 @@ impl ChatOptions {
382 }
383
384 /// Tool-choice semantics supported by `vllm-chat`.
385 -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
385 +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
386 #[serde(rename_all = "snake_case")]
387 pub enum ChatToolChoice {
388 - Auto,
388 #[default]
389 None,
390 + Auto,
391 + Required,
392 + Function {
393 + name: String,
394 + },
395 }
396
397 /// One chat request ready to be rendered into a prompt and lowered into a
@@ -486,7 +490,7 @@ impl ChatRequest {
490 /// Return true if this request should enable tool parsing based on the tool
491 /// choice and tool list.
492 pub(crate) fn tool_parsing_enabled(&self) -> bool {
489 - matches!(self.tool_choice, ChatToolChoice::Auto) && !self.tools.is_empty()
493 + !matches!(self.tool_choice, ChatToolChoice::None) && !self.tools.is_empty()
494 }
495
496 /// Return the request-level thinking toggle when explicitly requested.
rust/src/server/src/routes/openai/chat_completions/convert.rs
+75
@@ -361,6 +361,13 @@ fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoic
361 match tool_choice {
362 None | Some(ToolChoice::Value(ToolChoiceValue::Auto)) => Ok(ChatToolChoice::Auto),
363 Some(ToolChoice::Value(ToolChoiceValue::None)) => Ok(ChatToolChoice::None),
364 + Some(ToolChoice::Value(ToolChoiceValue::Required)) => Ok(ChatToolChoice::Required),
365 + Some(ToolChoice::Function {
366 + tool_type,
367 + function,
368 + }) if tool_type == "function" => Ok(ChatToolChoice::Function {
369 + name: function.name.clone(),
370 + }),
371 _ => bail_invalid_request!("tool_choice={:?} is not supported yet.", tool_choice),
372 }
373 }
@@ -962,6 +969,74 @@ mod tests {
969 assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::None);
970 }
971
972 + #[test]
973 + fn prepare_chat_request_lowers_required_tool_choice() {
974 + let request = ChatCompletionRequest {
975 + tools: Some(vec![Tool {
976 + tool_type: "function".to_string(),
977 + function: Function {
978 + name: "get_weather".to_string(),
979 + description: Some("Get weather".to_string()),
980 + parameters: json!({
981 + "type": "object",
982 + "properties": {"city": {"type": "string"}},
983 + }),
984 + strict: None,
985 + },
986 + }]),
987 + tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)),
988 + ..base_request()
989 + };
990 +
991 + let prepared = prepare_chat_request(
992 + request,
993 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
994 + ResolvedRequestContext::default(),
995 + )
996 + .expect("request is valid");
997 +
998 + assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Required);
999 + }
1000 +
1001 + #[test]
1002 + fn prepare_chat_request_lowers_named_function_tool_choice() {
1003 + let request = ChatCompletionRequest {
1004 + tools: Some(vec![Tool {
1005 + tool_type: "function".to_string(),
1006 + function: Function {
1007 + name: "get_weather".to_string(),
1008 + description: Some("Get weather".to_string()),
1009 + parameters: json!({
1010 + "type": "object",
1011 + "properties": {"city": {"type": "string"}},
1012 + }),
1013 + strict: None,
1014 + },
1015 + }]),
1016 + tool_choice: Some(ToolChoice::Function {
1017 + tool_type: "function".to_string(),
1018 + function: crate::routes::openai::utils::types::FunctionChoice {
1019 + name: "get_weather".to_string(),
1020 + },
1021 + }),
1022 + ..base_request()
1023 + };
1024 +
1025 + let prepared = prepare_chat_request(
1026 + request,
1027 + &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
1028 + ResolvedRequestContext::default(),
1029 + )
1030 + .expect("request is valid");
1031 +
1032 + assert_eq!(
1033 + prepared.chat_request.tool_choice,
1034 + ChatToolChoice::Function {
1035 + name: "get_weather".to_string(),
1036 + }
1037 + );
1038 + }
1039 +
1040 #[test]
1041 fn prepare_chat_request_lowers_logprobs_fields() {
1042 let request = ChatCompletionRequest {
rust/src/server/src/routes/openai/chat_completions/validate.rs
+2 -61
@@ -1,6 +1,6 @@
1 use super::types::ChatCompletionRequest;
2 use crate::error::{ApiError, bail_invalid_request};
3 -use crate::routes::openai::utils::types::{ChatMessage, Tool, ToolChoice, ToolChoiceValue};
3 +use crate::routes::openai::utils::types::{ChatMessage, Tool};
4
5 /// Enforce the minimal compatibility contract for the Rust OpenAI server.
6 pub(super) fn validate_request_compat(
@@ -58,30 +58,6 @@ pub(super) fn validate_request_compat(
58 }
59 }
60
61 - if let Some(tool_choice) = &request.tool_choice {
62 - match tool_choice {
63 - ToolChoice::Value(ToolChoiceValue::Auto | ToolChoiceValue::None) => {}
64 - ToolChoice::Value(ToolChoiceValue::Required) => {
65 - bail_invalid_request!(
66 - param = "tool_choice",
67 - "tool_choice=required is not supported yet."
68 - );
69 - }
70 - ToolChoice::Function { .. } => {
71 - bail_invalid_request!(
72 - param = "tool_choice",
73 - "Named function tool_choice is not supported yet."
74 - );
75 - }
76 - ToolChoice::AllowedTools { .. } => {
77 - bail_invalid_request!(
78 - param = "tool_choice",
79 - "allowed_tools tool_choice is not supported yet."
80 - );
81 - }
82 - }
83 - }
84 -
61 if request.use_beam_search {
62 bail_invalid_request!(
63 param = "use_beam_search",
@@ -159,8 +135,7 @@ mod tests {
135 use crate::routes::openai::chat_completions::types::ChatCompletionRequest;
136 use crate::routes::openai::utils::structured_outputs::ResponseFormat;
137 use crate::routes::openai::utils::types::{
162 - ChatMessage, Function, FunctionChoice, MessageContent, StringOrArray, Tool, ToolChoice,
163 - ToolChoiceValue, ToolReference,
138 + ChatMessage, Function, MessageContent, StringOrArray, Tool, ToolChoice, ToolChoiceValue,
139 };
140
141 fn served(names: &[&str]) -> Vec<String> {
@@ -357,38 +332,4 @@ mod tests {
332 validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"]))
333 .expect("tool_choice=none is ok");
334 }
360 -
361 - #[test]
362 - fn validate_request_compat_rejects_required_and_named_tool_choices() {
363 - let required = ChatCompletionRequest {
364 - tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)),
365 - ..base_request()
366 - };
367 - assert!(validate_request_compat(&required, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
368 -
369 - let named = ChatCompletionRequest {
370 - tool_choice: Some(ToolChoice::Function {
371 - tool_type: "function".to_string(),
372 - function: FunctionChoice {
373 - name: "tool".to_string(),
374 - },
375 - }),
376 - ..base_request()
377 - };
378 - assert!(validate_request_compat(&named, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err());
379 -
380 - let allowed_tools = ChatCompletionRequest {
381 - tool_choice: Some(ToolChoice::AllowedTools {
382 - tool_type: "allowed_tools".to_string(),
383 - mode: "auto".to_string(),
384 - tools: vec![ToolReference::Function {
385 - name: "tool".to_string(),
386 - }],
387 - }),
388 - ..base_request()
389 - };
390 - assert!(
391 - validate_request_compat(&allowed_tools, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()
392 - );
393 - }
335 }
rust/src/tool-parser/Cargo.toml
+1
@@ -14,6 +14,7 @@ serde_json.workspace = true
14 thiserror.workspace = true
15 thiserror-ext.workspace = true
16 winnow.workspace = true
17 +xgrammar-structural-tag.workspace = true
18
19 [dev-dependencies]
20 criterion.workspace = true
rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs
+5 -1
@@ -1,5 +1,5 @@
1 use super::{DeepSeekDsmlToolParser, DsmlTokens};
2 -use crate::{Result, Tool, ToolParser, ToolParserOutput};
2 +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4 /// Tool parser for DeepSeek V3.2 models.
5 ///
@@ -44,6 +44,10 @@ impl ToolParser for DeepSeekV32ToolParser {
44 true
45 }
46
47 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
48 + Some(StructuralTagModel::DeepSeekV32)
49 + }
50 +
51 fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
52 self.0.parse_into(chunk, output)
53 }
rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs
+16 -2
@@ -1,5 +1,5 @@
1 use super::{DeepSeekDsmlToolParser, DsmlTokens};
2 -use crate::{Result, Tool, ToolParser, ToolParserOutput};
2 +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4 /// Tool parser for DeepSeek V4 models.
5 ///
@@ -47,6 +47,10 @@ impl ToolParser for DeepSeekV4ToolParser {
47 true
48 }
49
50 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
51 + Some(StructuralTagModel::DeepSeekV4)
52 + }
53 +
54 fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
55 self.0.parse_into(chunk, output)
56 }
@@ -65,8 +69,8 @@ mod tests {
69 use serde_json::{Value, json};
70
71 use super::DeepSeekV4ToolParser;
68 - use crate::ToolParserTestExt as _;
72 use crate::test_utils::{collect_stream, test_tools};
73 + use crate::{StructuralTagModel, ToolParser, ToolParserTestExt as _};
74
75 fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
76 let params = params
@@ -83,6 +87,16 @@ mod tests {
87 )
88 }
89
90 + #[test]
91 + fn deepseek_v4_exposes_structural_tag_model() {
92 + let parser = DeepSeekV4ToolParser::new(&test_tools());
93 +
94 + assert_eq!(
95 + parser.structural_tag_model(),
96 + Some(StructuralTagModel::DeepSeekV4)
97 + );
98 + }
99 +
100 #[test]
101 fn deepseek_v4_parse_complete_reuses_dsml_parser_with_tool_calls_token() {
102 let mut parser = DeepSeekV4ToolParser::new(&test_tools());
rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs
+5 -1
@@ -1,5 +1,5 @@
1 use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
2 -use crate::{Result, Tool, ToolParser, ToolParserOutput};
2 +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4 /// Tool parser for DeepSeek V3 JSON-fenced tool calls.
5 ///
@@ -32,6 +32,10 @@ impl ToolParser for DeepSeekV3ToolParser {
32 Ok(Box::new(Self::new(tools)))
33 }
34
35 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
36 + Some(StructuralTagModel::DeepSeekR1)
37 + }
38 +
39 fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
40 self.0.parse_into(chunk, output)
41 }
rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs
+5 -1
@@ -1,5 +1,5 @@
1 use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser};
2 -use crate::{Result, Tool, ToolParser, ToolParserOutput};
2 +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4 /// Tool parser for DeepSeek V3.1 raw JSON tool calls.
5 ///
@@ -28,6 +28,10 @@ impl ToolParser for DeepSeekV31ToolParser {
28 Ok(Box::new(Self::new(tools)))
29 }
30
31 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
32 + Some(StructuralTagModel::DeepSeekV31)
33 + }
34 +
35 fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
36 self.0.parse_into(chunk, output)
37 }
rust/src/tool-parser/src/glm_xml/glm47_moe.rs
+5 -1
@@ -1,5 +1,5 @@
1 use super::{GlmXmlToolParser, Separator};
2 -use crate::{Result, Tool, ToolParser, ToolParserOutput};
2 +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4 /// Tool parser for GLM-4.7 MoE XML-style tool calls.
5 ///
@@ -22,6 +22,10 @@ impl ToolParser for Glm47MoeToolParser {
22 Ok(Box::new(Self::new(tools)))
23 }
24
25 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
26 + Some(StructuralTagModel::Glm47)
27 + }
28 +
29 fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
30 self.0.parse_into(chunk, output)
31 }
rust/src/tool-parser/src/hy_v3.rs
+5 -1
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
7 use super::parameters::ToolSchemas;
8 use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
9 use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
10 -use crate::Tool;
10 +use crate::{StructuralTagModel, Tool};
11
12 const TOOL_CALLS_START: &str = "<tool_calls>";
13 const TOOL_CALLS_END: &str = "</tool_calls>";
@@ -113,6 +113,10 @@ impl ToolParser for HyV3ToolParser {
113 Ok(Box::new(Self::new(tools)))
114 }
115
116 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
117 + Some(StructuralTagModel::HyV3)
118 + }
119 +
120 fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
121 self.buffer.push_str(chunk);
122
rust/src/tool-parser/src/json/hermes.rs
+5 -1
@@ -1,5 +1,5 @@
1 use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
2 -use crate::{Result, Tool, ToolParser, ToolParserOutput};
2 +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4 const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
5 parser_name: "Hermes",
@@ -45,6 +45,10 @@ impl ToolParser for HermesToolParser {
45 Ok(Box::new(Self::new(tools)))
46 }
47
48 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
49 + Some(StructuralTagModel::Hermes)
50 + }
51 +
52 fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
53 self.inner.parse_into(chunk, output)
54 }
rust/src/tool-parser/src/json/llama.rs
+5 -1
@@ -9,7 +9,7 @@ use super::{
9 argument_delta_event, tool_call_header_event,
10 };
11 use crate::utils::{JsonObjectScanState, parse_buffered_event};
12 -use crate::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
12 +use crate::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput};
13
14 #[derive(Debug, Clone, PartialEq, Eq)]
15 enum LlamaJsonMode {
@@ -133,6 +133,10 @@ impl ToolParser for Llama3JsonToolParser {
133 Ok(Box::new(Self::new(tools)))
134 }
135
136 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
137 + Some(StructuralTagModel::Llama)
138 + }
139 +
140 fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
141 self.buffer.push_str(chunk);
142
rust/src/tool-parser/src/json/qwen.rs
+5 -1
@@ -1,5 +1,5 @@
1 use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
2 -use crate::{Result, Tool, ToolParser, ToolParserOutput};
2 +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput};
3
4 const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
5 parser_name: "Qwen XML",
@@ -47,6 +47,10 @@ impl ToolParser for Qwen3XmlToolParser {
47 Ok(Box::new(Self::new(tools)))
48 }
49
50 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
51 + Some(StructuralTagModel::Qwen3)
52 + }
53 +
54 fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
55 self.inner.parse_into(chunk, output)
56 }
rust/src/tool-parser/src/kimi_k2.rs
+5 -1
@@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until, take_while};
8
9 use super::utils::{JsonObjectScanState, parse_buffered_event, safe_text_len, take_json_object};
10 use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
11 -use crate::Tool;
11 +use crate::{StructuralTagModel, Tool};
12
13 const TOOL_CALLS_START: &str = "<|tool_calls_section_begin|>";
14 const TOOL_CALLS_END: &str = "<|tool_calls_section_end|>";
@@ -147,6 +147,10 @@ impl ToolParser for KimiK2ToolParser {
147 true
148 }
149
150 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
151 + Some(StructuralTagModel::Kimi)
152 + }
153 +
154 fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
155 self.call_ids.get(&tool_index).map(String::as_str)
156 }
rust/src/tool-parser/src/lib.rs
+6
@@ -35,6 +35,7 @@ pub use minimax_m3::MinimaxM3ToolParser;
35 pub use qwen_coder::Qwen3CoderToolParser;
36 use serde::{Deserialize, Serialize};
37 use serde_json::Value;
38 +pub use xgrammar_structural_tag::Model as StructuralTagModel;
39
40 /// One function-style tool made available to the model.
41 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -123,6 +124,11 @@ pub trait ToolParser: Send {
124 false
125 }
126
127 + /// Return the xgrammar structural-tag model used for strict tool calling.
128 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
129 + None
130 + }
131 +
132 /// Return the parser-provided ID for a tool call by index, if the model
133 /// emitted one.
134 fn tool_call_id(&self, _tool_index: usize) -> Option<&str> {
rust/src/tool-parser/src/minimax_m2.rs
+5 -1
@@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until};
7 use super::parameters::ToolSchemas;
8 use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
9 use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
10 -use crate::Tool;
10 +use crate::{StructuralTagModel, Tool};
11
12 const TOOL_CALL_START: &str = "<minimax:tool_call>";
13 const TOOL_CALL_END: &str = "</minimax:tool_call>";
@@ -112,6 +112,10 @@ impl ToolParser for MinimaxM2ToolParser {
112 Ok(Box::new(Self::new(tools)))
113 }
114
115 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
116 + Some(StructuralTagModel::Minimax)
117 + }
118 +
119 fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
120 self.buffer.push_str(chunk);
121
rust/src/tool-parser/src/qwen_coder.rs
+16 -2
@@ -6,7 +6,7 @@ use winnow::token::{literal, take_until};
6
7 use super::parameters::ToolSchemas;
8 use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
9 -use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
9 +use super::{Result, StructuralTagModel, ToolCallDelta, ToolParser, ToolParserOutput};
10 use crate::Tool;
11
12 const TOOL_CALL_START: &str = "<tool_call>";
@@ -113,6 +113,10 @@ impl ToolParser for Qwen3CoderToolParser {
113 Ok(Box::new(Self::new(tools)))
114 }
115
116 + fn structural_tag_model(&self) -> Option<StructuralTagModel> {
117 + Some(StructuralTagModel::Qwen3Coder)
118 + }
119 +
120 fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
121 self.buffer.push_str(chunk);
122
@@ -236,7 +240,7 @@ mod tests {
240 use serde_json::{Value, json};
241 use thiserror_ext::AsReport;
242
239 - use super::{Qwen3CoderToolParser, ToolParser};
243 + use super::{Qwen3CoderToolParser, StructuralTagModel, ToolParser};
244 use crate::test_utils::{collect_stream, split_by_chars, test_tools};
245 use crate::{ToolParserOutput, ToolParserTestExt as _};
246
@@ -249,6 +253,16 @@ mod tests {
253 format!("<tool_call>\n<function={function_name}>\n{params}\n</function>\n</tool_call>")
254 }
255
256 + #[test]
257 + fn qwen_coder_exposes_structural_tag_model() {
258 + let parser = Qwen3CoderToolParser::new(&test_tools());
259 +
260 + assert_eq!(
261 + parser.structural_tag_model(),
262 + Some(StructuralTagModel::Qwen3Coder)
263 + );
264 + }
265 +
266 #[test]
267 fn qwen_coder_parse_complete_without_tool_call_keeps_text() {
268 let mut parser = Qwen3CoderToolParser::new(&test_tools());