@tduncan / Nvidia-Optimization-Stack-VLLM / commits / d50723df04

[Rust Frontend] Introduce parser-owned output grammar interfaces (#55269)

Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

Bugen Zhao committed Sep 22, 2026 at 13:35 UTC d50723df04f77c732c75e07d337ee81d843e014f
34 files changed +814 -698
rust/Cargo.lock
+1
@@ -5937,6 +5937,7 @@ dependencies = [
5937 "openai-protocol",
5938 "serde",
5939 "serde_json",
5940 + "serde_with",
5941 "thiserror 2.0.18",
5942 "thiserror-ext",
5943 "tool-parser",
rust/src/chat/src/error.rs
+10 -2
@@ -60,6 +60,11 @@ pub enum Error {
60 #[source]
61 error: BoxedError,
62 },
63 + #[error("failed to initialize request output parser")]
64 + OutputParserInitialization {
65 + #[source]
66 + error: BoxedError,
67 + },
68 #[error(
69 "gpt_oss uses native Harmony output parsing; generic {kind} parser override `{selection}` is not supported"
70 )]
@@ -87,8 +92,11 @@ pub enum Error {
92 ToolChoiceRequiresTools,
93 #[error("tool_choice function `{name}` was not found in the available tools")]
94 ToolChoiceFunctionNotFound { name: String },
90 - #[error("failed to build structural tag: {message}")]
91 - StructuralTag { message: String },
95 + #[error("failed to build output grammar")]
96 + OutputGrammar {
97 + #[source]
98 + error: BoxedError,
99 + },
100 #[error(transparent)]
101 Text(#[from] vllm_text::Error),
102 #[error(transparent)]
rust/src/chat/src/lib.rs
+29 -14
@@ -22,6 +22,7 @@ pub use event::{
22 };
23 use futures::{StreamExt, TryStreamExt as _};
24 pub use llm_multimodal::MediaContentPart;
25 +use output::apply_output_grammar;
26 pub use output::{
27 ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor,
28 HarmonyChatOutputProcessor,
@@ -65,7 +66,7 @@ use vllm_engine_core_client::EngineCoreClient;
66 use vllm_engine_core_client::protocol::dtype::ModelDtype;
67 use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
68 use vllm_llm::Llm;
68 -use vllm_text::{Prompt, TextLlm, TextRequest};
69 +use vllm_text::{Prompt, TextLlm, TextRequest, TextRequestProcessor};
70
71 /// Chat request preparation shared by inference and render-only frontends.
72 pub struct ChatRequestProcessor {
@@ -174,15 +175,8 @@ impl ChatRequestProcessor {
175 async fn prepare_text_request(&self, request: ChatRequest) -> Result<TextRequest> {
176 // Stamp before rendering so render and tokenize count toward TTFT/e2e.
177 let arrival_time = vllm_llm::current_unix_timestamp_secs();
177 - let rendered = self.backend.chat_renderer().render(&request)?;
178 - let reasoning_parser_kwargs =
179 - request
180 - .sampling_params
181 - .structured_outputs
182 - .is_some()
183 - .then(|| ReasoningParserKwargs {
184 - chat_template_kwargs: rendered.effective_template_kwargs.clone(),
185 - });
178 + let mut rendered = self.backend.chat_renderer().render(&request)?;
179 + let chat_template_kwargs = std::mem::take(&mut rendered.effective_template_kwargs);
180 let (prompt, mm_features) = self.finalize_rendered_prompt(&request, rendered).await?;
181 Ok(TextRequest {
182 request_id: request.request_id,
@@ -198,7 +192,10 @@ impl ChatRequestProcessor {
192 data_parallel_rank: request.data_parallel_rank,
193 session_id: request.session_id,
194 kv_hints: None,
201 - reasoning_parser_kwargs,
195 + reasoning_parser_kwargs: ReasoningParserKwargs {
196 + chat_template_kwargs,
197 + },
198 + reasoning_ended: None,
199 lora_request: request.lora_request,
200 arrival_time: Some(arrival_time),
201 })
@@ -211,12 +208,19 @@ impl ChatRequestProcessor {
208 }
209
210 /// Prepare one chat request without submitting it to an engine.
211 + ///
212 + /// The returned text request is already tokenized: the output processor
213 + /// is initialized from the final prompt token IDs and its grammar, if any,
214 + /// is applied to the request before it is handed to `text_processor` again
215 + /// for lowering.
216 pub async fn prepare(
217 &self,
218 mut request: ChatRequest,
219 + text_processor: &TextRequestProcessor,
220 ) -> Result<(TextRequest, DynChatOutputProcessor)> {
221 request.validate()?;
219 - let output_processor = self.backend.new_chat_output_processor(
222 +
223 + let mut output_processor = self.backend.new_chat_output_processor(
224 &mut request,
225 NewChatOutputProcessorOptions {
226 tool_call_parser: &self.tool_call_parser,
@@ -224,7 +228,17 @@ impl ChatRequestProcessor {
228 tool_strict_level: self.tool_strict_level,
229 },
230 )?;
227 - let text_request = self.prepare_text_request(request).await?;
231 + let mut text_request = self.prepare_text_request(request).await?;
232 +
233 + // Initialize the output processor from the final prompt token IDs and apply
234 + // the output grammar to the request, if any.
235 + let prompt_token_ids = text_processor.tokenize_in_place(&mut text_request)?;
236 + let grammar = {
237 + output_processor.initialize(prompt_token_ids)?;
238 + output_processor.build_output_grammar()?
239 + };
240 + apply_output_grammar(&mut text_request, grammar)?;
241 +
242 Ok((text_request, output_processor))
243 }
244 }
@@ -342,7 +356,8 @@ impl ChatLlm {
356
357 /// Render, tokenize, and submit one chat request.
358 pub async fn chat(&self, request: ChatRequest) -> Result<ChatEventStream> {
345 - let (text_request, output_processor) = self.processor.prepare(request).await?;
359 + let (text_request, output_processor) =
360 + self.processor.prepare(request, self.text.request_processor()).await?;
361 let request_id = text_request.request_id.clone();
362 let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();
363
rust/src/chat/src/output/default/mod.rs
+125 -10
@@ -3,17 +3,17 @@
3
4 //! Default output processing pipeline.
5
6 -mod structural_tag;
6 mod unified;
7
8 use std::sync::Once;
9
10 use futures::StreamExt as _;
11 use tracing::info;
12 +use vllm_parser::output_grammar::{BuiltOutputGrammar, OutputGrammarContext};
13 use vllm_parser::unified::{CombinedParser, UnifiedParser};
14 use vllm_text::tokenizer::DynTokenizer;
15 +use xgrammar_structural_tag::ToolChoice;
16
16 -use self::structural_tag::apply_structural_tag_constraint;
17 use self::unified::unified_event_stream;
18 use super::structured::structured_chat_event_stream;
19 use crate::error::Result;
@@ -34,6 +34,18 @@ use crate::{Error, Result as ChatResult};
34 pub struct DefaultChatOutputProcessor {
35 parser: Box<dyn UnifiedParser>,
36 parallel_tool_calls: bool,
37 + /// Request facts the initialized parser needs to build its output grammar.
38 + /// Absent for the plain-text-only processor, which never builds one.
39 + grammar_inputs: Option<GrammarInputs>,
40 +}
41 +
42 +/// Request-scoped inputs for [`UnifiedParser::build_output_grammar`], captured
43 +/// at construction because the chat request is consumed before the prompt is
44 +/// tokenized.
45 +struct GrammarInputs {
46 + tools: Vec<ChatTool>,
47 + tool_choice: ToolChoice,
48 + tool_strict_level: ToolStrictLevel,
49 }
50
51 impl DefaultChatOutputProcessor {
@@ -42,7 +54,8 @@ impl DefaultChatOutputProcessor {
54 ///
55 /// Parser resolution happens here so that request validation, prompt
56 /// rendering, and streaming all observe the same parser-adjusted
45 - /// request state.
57 + /// request state. The parser is initialized and its output grammar built
58 + /// later, once the final prompt token IDs are known.
59 pub fn new(
60 request: &mut ChatRequest,
61 model_id: &str,
@@ -74,12 +87,6 @@ impl DefaultChatOutputProcessor {
87 Box::new(CombinedParser::new(reasoning_parser, tool_parser)) as Box<dyn UnifiedParser>
88 };
89
77 - apply_structural_tag_constraint(
78 - request,
79 - parser.structural_tag_builder(),
80 - tool_strict_level,
81 - )?;
82 -
90 if parser.preserve_special_tokens() {
91 request.decode_options.skip_special_tokens = false;
92 }
@@ -87,6 +94,11 @@ impl DefaultChatOutputProcessor {
94 Ok(Self {
95 parser,
96 parallel_tool_calls: request.parallel_tool_calls(),
97 + grammar_inputs: Some(GrammarInputs {
98 + tools: request.tools().to_vec(),
99 + tool_choice: request.tool_choice().into(),
100 + tool_strict_level,
101 + }),
102 })
103 }
104
@@ -99,6 +111,7 @@ impl DefaultChatOutputProcessor {
111 Self {
112 parser: Box::new(CombinedParser::plain_text_only()),
113 parallel_tool_calls: true,
114 + grammar_inputs: None,
115 }
116 }
117
@@ -175,6 +188,30 @@ static REASONING_PARSER_LOG_ONCE: Once = Once::new();
188 static UNIFIED_PARSER_LOG_ONCE: Once = Once::new();
189
190 impl ChatOutputProcessor for DefaultChatOutputProcessor {
191 + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
192 + self.parser.initialize(prompt_token_ids).map_err(|error| {
193 + Error::OutputParserInitialization {
194 + error: Box::new(error),
195 + }
196 + })
197 + }
198 +
199 + fn build_output_grammar(&self) -> Result<Option<BuiltOutputGrammar>> {
200 + let Some(inputs) = &self.grammar_inputs else {
201 + return Ok(None);
202 + };
203 + self.parser
204 + .build_output_grammar(&OutputGrammarContext {
205 + tools: &inputs.tools,
206 + tool_choice: &inputs.tool_choice,
207 + tool_strict_level: inputs.tool_strict_level,
208 + parallel_tool_calls: self.parallel_tool_calls,
209 + })
210 + .map_err(|error| Error::OutputGrammar {
211 + error: Box::new(error),
212 + })
213 + }
214 +
215 /// Transforms a raw generate-output token stream into structured chat
216 /// events through two sequential stages once text decoding has
217 /// already happened:
@@ -196,8 +233,9 @@ mod tests {
233 use vllm_tokenizer::test_utils::TestTokenizer;
234
235 use super::DefaultChatOutputProcessor;
236 + use crate::output::ChatOutputProcessor;
237 use crate::parser::{ParserSelection, ToolStrictLevel};
200 - use crate::request::ChatRequest;
238 + use crate::request::{ChatRequest, ChatTool, ChatToolChoice, ResolvedToolContext};
239
240 fn tokenizer() -> Arc<TestTokenizer> {
241 Arc::new(
@@ -207,6 +245,83 @@ mod tests {
245 )
246 }
247
248 + #[test]
249 + fn output_grammar_preserves_tool_strict_level() {
250 + let build = |level, strict, choice| {
251 + let tools = vec![ChatTool {
252 + name: "search".to_string(),
253 + description: None,
254 + parameters: serde_json::json!({
255 + "type": "object",
256 + "properties": { "query": { "type": "string" } },
257 + "required": ["query"]
258 + }),
259 + strict,
260 + }];
261 + let mut request = ChatRequest {
262 + tool_context: ResolvedToolContext::new(&[], tools, Some(choice), true).unwrap(),
263 + ..ChatRequest::for_test()
264 + };
265 + let mut processor = DefaultChatOutputProcessor::new(
266 + &mut request,
267 + "other-model",
268 + tokenizer(),
269 + &ParserSelection::Explicit("qwen3_coder".to_string()),
270 + &ParserSelection::None,
271 + level,
272 + )
273 + .unwrap();
274 + processor.initialize(&[]).unwrap();
275 + processor.build_output_grammar().unwrap()
276 + };
277 +
278 + assert!(build(ToolStrictLevel::Auto, None, ChatToolChoice::Auto).is_none());
279 + let envelope = build(ToolStrictLevel::Function, None, ChatToolChoice::Auto).unwrap();
280 + let parameters = build(ToolStrictLevel::Parameter, None, ChatToolChoice::Auto).unwrap();
281 + let strict = build(ToolStrictLevel::Auto, Some(true), ChatToolChoice::Auto).unwrap();
282 + assert_eq!(parameters, strict);
283 + assert_ne!(envelope, parameters);
284 + assert!(build(ToolStrictLevel::Parameter, None, ChatToolChoice::None).is_none());
285 + }
286 +
287 + #[test]
288 + fn tool_grammar_honors_parallel_call_policy() {
289 + for parallel in [false, true] {
290 + let tools = vec![ChatTool {
291 + name: "lookup".to_string(),
292 + description: None,
293 + parameters: serde_json::json!({"type": "object"}),
294 + strict: Some(true),
295 + }];
296 + let mut request = ChatRequest {
297 + tool_context: ResolvedToolContext::new(
298 + &[],
299 + tools,
300 + Some(ChatToolChoice::Auto),
301 + parallel,
302 + )
303 + .unwrap(),
304 + ..ChatRequest::for_test()
305 + };
306 + let mut processor = DefaultChatOutputProcessor::new(
307 + &mut request,
308 + "other-model",
309 + tokenizer(),
310 + &ParserSelection::Explicit("qwen3_coder".to_string()),
311 + &ParserSelection::None,
312 + ToolStrictLevel::Auto,
313 + )
314 + .unwrap();
315 + processor.initialize(&[]).unwrap();
316 + let built = processor.build_output_grammar().unwrap().unwrap();
317 + let tag =
318 + serde_json::to_value(xgrammar_structural_tag::StructuralTag::new(built.format))
319 + .unwrap();
320 + assert_eq!(tag["format"]["type"], "triggered_tags");
321 + assert_eq!(tag["format"]["stop_after_first"], !parallel);
322 + }
323 + }
324 +
325 #[test]
326 fn equal_explicit_gemma4_uses_unified_parser() {
327 let mut request = ChatRequest::for_test();
rust/src/chat/src/output/default/structural_tag.rs deleted
-507
@@ -1,507 +0,0 @@
1 -// SPDX-License-Identifier: Apache-2.0
2 -// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -//! Applies xgrammar structural-tag constraints for strict tool calling.
5 -
6 -use thiserror_ext::AsReport;
7 -use vllm_engine_core_client::protocol::structured_outputs::{
8 - StructuredOutputBackend, StructuredOutputsParams,
9 -};
10 -use vllm_parser::tool::StructuralTagBuilder;
11 -use xgrammar_structural_tag::builders::StructuralTagOptions;
12 -use xgrammar_structural_tag::{
13 - FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam,
14 - build_structural_tag,
15 -};
16 -
17 -use crate::parser::ToolStrictLevel;
18 -use crate::request::{ChatRequest, ChatToolChoice};
19 -use crate::{Error, Result as ChatResult};
20 -
21 -/// Apply structural tag constraints to the request based on the tool parser's structural tag
22 -/// support, the request's tool choice, and the server-side strictness floor.
23 -pub(super) fn apply_structural_tag_constraint(
24 - request: &mut ChatRequest,
25 - builder: Option<&dyn StructuralTagBuilder>,
26 - strict_level: ToolStrictLevel,
27 -) -> ChatResult<()> {
28 - let Some(builder) = builder else {
29 - return Ok(());
30 - };
31 - let Some(tool_choice) = structural_tag_tool_choice(request, strict_level) else {
32 - return Ok(());
33 - };
34 -
35 - let tools = request
36 - .tools()
37 - .iter()
38 - .map(|tool| {
39 - // A tool without an explicit `strict` is non-strict: its call envelope is
40 - // constrained, but its arguments stay free unless the server level is `parameter`.
41 - let strict = strict_level >= ToolStrictLevel::Parameter || tool.strict == Some(true);
42 - ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
43 - name: tool.name.clone(),
44 - description: tool.description.clone(),
45 - parameters: Some(tool.parameters.clone()),
46 - strict: Some(strict),
47 - }))
48 - })
49 - .collect::<Vec<_>>();
50 -
51 - let structural_tag = build_structural_tag(
52 - builder,
53 - &tools,
54 - tool_choice,
55 - StructuralTagOptions::default()
56 - .with_reasoning(false)
57 - .with_parallel_tool_calls(request.parallel_tool_calls()),
58 - )
59 - .and_then(|tag| tag.to_json_string())
60 - .map_err(|error| Error::StructuralTag {
61 - message: error.to_report_string(),
62 - })?;
63 -
64 - // Overwrite any existing structured output settings with the structural tag constraint.
65 - request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
66 - backend: StructuredOutputBackend::Xgrammar,
67 - ..StructuredOutputsParams::structural_tag(structural_tag)
68 - });
69 -
70 - Ok(())
71 -}
72 -
73 -/// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request.
74 -///
75 -/// Returns `None` if no structural tag constraints should be applied.
76 -fn structural_tag_tool_choice(
77 - request: &ChatRequest,
78 - strict_level: ToolStrictLevel,
79 -) -> Option<StructuralTagToolChoice> {
80 - if request.tools().is_empty() {
81 - return None;
82 - }
83 -
84 - match request.tool_choice() {
85 - ChatToolChoice::None => None,
86 - // For `Auto`, apply the structural tag only when the server raises the floor or at
87 - // least one tool opts in with `strict: true`.
88 - ChatToolChoice::Auto => (strict_level >= ToolStrictLevel::Function
89 - || request.tools().iter().any(|tool| tool.strict == Some(true)))
90 - .then(StructuralTagToolChoice::auto),
91 -
92 - ChatToolChoice::Required => Some(StructuralTagToolChoice::required()),
93 - ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())),
94 - }
95 -}
96 -
97 -#[cfg(test)]
98 -mod tests {
99 - use serde_json::{Value, json};
100 - use vllm_engine_core_client::protocol::structured_outputs::{
101 - StructuredOutputBackend, StructuredOutputsParams,
102 - };
103 - use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
104 -
105 - use super::*;
106 - use crate::request::{ChatMessage, ResolvedToolContext};
107 -
108 - fn chat_tool(name: &str, strict: Option<bool>) -> Tool {
109 - Tool {
110 - name: name.to_string(),
111 - description: None,
112 - parameters: json!({
113 - "type": "object",
114 - "properties": {
115 - "query": { "type": "string" }
116 - },
117 - "required": ["query"]
118 - }),
119 - strict,
120 - }
121 - }
122 -
123 - fn qwen3_coder_parser(tools: &[Tool]) -> Box<dyn ToolParser> {
124 - Qwen3CoderToolParser::create(tools).expect("Qwen3 Coder parser should build")
125 - }
126 -
127 - fn request(tool_choice: ChatToolChoice, tools: Vec<Tool>) -> ChatRequest {
128 - ChatRequest {
129 - tool_context: ResolvedToolContext::new(&[], tools, Some(tool_choice), true)
130 - .expect("tool context should resolve"),
131 - ..ChatRequest::for_test()
132 - }
133 - }
134 -
135 - fn structural_tag_value(request: &ChatRequest) -> Value {
136 - let params = request
137 - .sampling_params
138 - .structured_outputs
139 - .as_ref()
140 - .expect("structured outputs should be set");
141 - assert_eq!(params.backend, StructuredOutputBackend::Xgrammar);
142 - let structural_tag = params
143 - .constraint
144 - .as_structural_tag()
145 - .expect("structured output constraint should be structural_tag");
146 - serde_json::from_str(structural_tag).expect("structural_tag should be valid JSON")
147 - }
148 -
149 - fn structured_outputs(request: &ChatRequest) -> &StructuredOutputsParams {
150 - request
151 - .sampling_params
152 - .structured_outputs
153 - .as_ref()
154 - .expect("structured outputs should be set")
155 - }
156 -
157 - #[test]
158 - fn tool_grammar_honors_parallel_call_policy() {
159 - for parallel in [false, true] {
160 - let tools = vec![chat_tool("lookup", Some(true))];
161 - let mut request = ChatRequest {
162 - tool_context: ResolvedToolContext::new(
163 - &[],
164 - tools,
165 - Some(ChatToolChoice::Auto),
166 - parallel,
167 - )
168 - .unwrap(),
169 - ..ChatRequest::for_test()
170 - };
171 - let parser = qwen3_coder_parser(request.tools());
172 - apply_structural_tag_constraint(
173 - &mut request,
174 - parser.structural_tag_builder(),
175 - ToolStrictLevel::Auto,
176 - )
177 - .unwrap();
178 - let tag = structural_tag_value(&request);
179 - assert_eq!(tag["format"]["type"], "triggered_tags");
180 - assert_eq!(tag["format"]["stop_after_first"], !parallel);
181 - }
182 - }
183 -
184 - #[test]
185 - fn auto_strict_tool_choice_builds_structural_tag() {
186 - let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
187 - let parser = qwen3_coder_parser(request.tools());
188 -
189 - apply_structural_tag_constraint(
190 - &mut request,
191 - parser.structural_tag_builder(),
192 - ToolStrictLevel::Auto,
193 - )
194 - .expect("structural tag should build");
195 -
196 - let tag = structural_tag_value(&request);
197 - assert_eq!(tag["type"], "structural_tag");
198 - assert!(tag.to_string().contains("search"));
199 - }
200 -
201 - #[test]
202 - fn required_dynamic_tool_builds_structural_tag_from_effective_tools() {
203 - let messages = vec![ChatMessage::developer(
204 - "",
205 - Some(vec![chat_tool("lookup", None)]),
206 - )];
207 - let tool_context =
208 - ResolvedToolContext::new(&messages, Vec::new(), Some(ChatToolChoice::Required), true)
209 - .expect("dynamic tool context should resolve");
210 - let mut request = ChatRequest {
211 - messages,
212 - tool_context,
213 - ..ChatRequest::for_test()
214 - };
215 - let parser = qwen3_coder_parser(request.tools());
216 -
217 - apply_structural_tag_constraint(
218 - &mut request,
219 - parser.structural_tag_builder(),
220 - ToolStrictLevel::Auto,
221 - )
222 - .expect("structural tag should build");
223 -
224 - let tag = structural_tag_value(&request);
225 - assert!(tag.to_string().contains("lookup"));
226 - }
227 -
228 - #[test]
229 - fn required_initial_and_dynamic_tools_build_one_structural_tag() {
230 - let messages = vec![ChatMessage::developer(
231 - "",
232 - Some(vec![chat_tool("lookup", None)]),
233 - )];
234 - let tool_context = ResolvedToolContext::new(
235 - &messages,
236 - vec![chat_tool("search", None)],
237 - Some(ChatToolChoice::Required),
238 - true,
239 - )
240 - .expect("tool context should resolve");
241 - let mut request = ChatRequest {
242 - messages,
243 - tool_context,
244 - ..ChatRequest::for_test()
245 - };
246 - let parser = qwen3_coder_parser(request.tools());
247 -
248 - apply_structural_tag_constraint(
249 - &mut request,
250 - parser.structural_tag_builder(),
251 - ToolStrictLevel::Auto,
252 - )
253 - .expect("structural tag should build");
254 -
255 - let tag = structural_tag_value(&request).to_string();
256 - assert!(tag.contains("search"));
257 - assert!(tag.contains("lookup"));
258 - }
259 -
260 - #[test]
261 - fn auto_non_strict_tool_choice_skips_structural_tag() {
262 - let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
263 - let parser = qwen3_coder_parser(request.tools());
264 -
265 - apply_structural_tag_constraint(
266 - &mut request,
267 - parser.structural_tag_builder(),
268 - ToolStrictLevel::Auto,
269 - )
270 - .expect("structural tag decision should succeed");
271 -
272 - assert!(request.sampling_params.structured_outputs.is_none());
273 - }
274 -
275 - #[test]
276 - fn auto_strict_tool_choice_overwrites_existing_json_guidance() {
277 - let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
278 - request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
279 - backend: StructuredOutputBackend::Xgrammar,
280 - ..StructuredOutputsParams::json(json!({"type": "object"}))
281 - });
282 - let parser = qwen3_coder_parser(request.tools());
283 -
284 - apply_structural_tag_constraint(
285 - &mut request,
286 - parser.structural_tag_builder(),
287 - ToolStrictLevel::Auto,
288 - )
289 - .expect("structural tag should build");
290 -
291 - let params = structured_outputs(&request);
292 - assert!(params.constraint.is_structural_tag());
293 - let tag = structural_tag_value(&request);
294 - assert_eq!(tag["type"], "structural_tag");
295 - assert!(tag.to_string().contains("search"));
296 - }
297 -
298 - #[test]
299 - fn required_tool_choice_builds_structural_tag_without_strict_tools() {
300 - let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
301 - let parser = qwen3_coder_parser(request.tools());
302 -
303 - apply_structural_tag_constraint(
304 - &mut request,
305 - parser.structural_tag_builder(),
306 - ToolStrictLevel::Auto,
307 - )
308 - .expect("structural tag should build");
309 -
310 - let tag = structural_tag_value(&request);
311 - assert_eq!(tag["type"], "structural_tag");
312 - assert!(tag.to_string().contains("search"));
313 - }
314 -
315 - #[test]
316 - fn required_tool_choice_overwrites_existing_json_object_guidance() {
317 - let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
318 - request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
319 - backend: StructuredOutputBackend::Xgrammar,
320 - ..StructuredOutputsParams::json_object()
321 - });
322 - let parser = qwen3_coder_parser(request.tools());
323 -
324 - apply_structural_tag_constraint(
325 - &mut request,
326 - parser.structural_tag_builder(),
327 - ToolStrictLevel::Auto,
328 - )
329 - .expect("structural tag should build");
330 -
331 - let params = structured_outputs(&request);
332 - assert!(params.constraint.is_structural_tag());
333 - let tag = structural_tag_value(&request);
334 - assert_eq!(tag["type"], "structural_tag");
335 - assert!(tag.to_string().contains("search"));
336 - }
337 -
338 - #[test]
339 - fn named_tool_choice_builds_structural_tag_for_named_tool_only() {
340 - let mut request = request(
341 - ChatToolChoice::Function {
342 - name: "lookup".to_string(),
343 - },
344 - vec![chat_tool("search", None), chat_tool("lookup", None)],
345 - );
346 - let parser = qwen3_coder_parser(request.tools());
347 -
348 - apply_structural_tag_constraint(
349 - &mut request,
350 - parser.structural_tag_builder(),
351 - ToolStrictLevel::Auto,
352 - )
353 - .expect("structural tag should build");
354 -
355 - let tag = structural_tag_value(&request).to_string();
356 - assert!(tag.contains("lookup"));
357 - assert!(!tag.contains("search"));
358 - }
359 -
360 - #[test]
361 - fn none_tool_choice_skips_structural_tag() {
362 - let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
363 - let parser = qwen3_coder_parser(request.tools());
364 -
365 - apply_structural_tag_constraint(
366 - &mut request,
367 - parser.structural_tag_builder(),
368 - ToolStrictLevel::Auto,
369 - )
370 - .expect("structural tag decision should succeed");
371 -
372 - assert!(request.sampling_params.structured_outputs.is_none());
373 - }
374 -
375 - #[test]
376 - fn none_tool_choice_preserves_existing_json_object_guidance() {
377 - let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
378 - request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
379 - backend: StructuredOutputBackend::Xgrammar,
380 - ..StructuredOutputsParams::json_object()
381 - });
382 - let parser = qwen3_coder_parser(request.tools());
383 -
384 - apply_structural_tag_constraint(
385 - &mut request,
386 - parser.structural_tag_builder(),
387 - ToolStrictLevel::Auto,
388 - )
389 - .expect("structural tag decision should succeed");
390 -
391 - let params = structured_outputs(&request);
392 - assert!(params.constraint.is_json_object());
393 - }
394 -
395 - fn build(tool_choice: ChatToolChoice, tools: Vec<Tool>, level: ToolStrictLevel) -> ChatRequest {
396 - let mut request = request(tool_choice, tools);
397 - let parser = qwen3_coder_parser(request.tools());
398 -
399 - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder(), level)
400 - .expect("structural tag decision should succeed");
401 - request
402 - }
403 -
404 - fn tag_string(request: &ChatRequest) -> String {
405 - structural_tag_value(request).to_string()
406 - }
407 -
408 - #[test]
409 - fn function_level_constrains_auto_without_strict_tools() {
410 - let request = build(
411 - ChatToolChoice::Auto,
412 - vec![chat_tool("search", None)],
413 - ToolStrictLevel::Function,
414 - );
415 -
416 - let tag = structural_tag_value(&request);
417 - assert_eq!(tag["type"], "structural_tag");
418 - assert!(tag.to_string().contains("search"));
419 - }
420 -
421 - #[test]
422 - fn absent_strict_is_non_strict() {
423 - let unset = build(
424 - ChatToolChoice::Required,
425 - vec![chat_tool("search", None)],
426 - ToolStrictLevel::Auto,
427 - );
428 - let non_strict = build(
429 - ChatToolChoice::Required,
430 - vec![chat_tool("search", Some(false))],
431 - ToolStrictLevel::Auto,
432 - );
433 - let strict = build(
434 - ChatToolChoice::Required,
435 - vec![chat_tool("search", Some(true))],
436 - ToolStrictLevel::Auto,
437 - );
438 -
439 - assert_eq!(tag_string(&unset), tag_string(&non_strict));
440 - assert_ne!(tag_string(&unset), tag_string(&strict));
441 - }
442 -
443 - #[test]
444 - fn strict_tool_does_not_pin_its_neighbour() {
445 - let mixed = build(
446 - ChatToolChoice::Auto,
447 - vec![chat_tool("weather", Some(true)), chat_tool("search", None)],
448 - ToolStrictLevel::Auto,
449 - );
450 - let explicit = build(
451 - ChatToolChoice::Auto,
452 - vec![
453 - chat_tool("weather", Some(true)),
454 - chat_tool("search", Some(false)),
455 - ],
456 - ToolStrictLevel::Auto,
457 - );
458 -
459 - assert_eq!(tag_string(&mixed), tag_string(&explicit));
460 - }
461 -
462 - #[test]
463 - fn parameter_level_pins_argument_schemas() {
464 - let pinned = build(
465 - ChatToolChoice::Auto,
466 - vec![chat_tool("search", None)],
467 - ToolStrictLevel::Parameter,
468 - );
469 - let strict = build(
470 - ChatToolChoice::Auto,
471 - vec![chat_tool("search", Some(true))],
472 - ToolStrictLevel::Auto,
473 - );
474 - let envelope_only = build(
475 - ChatToolChoice::Auto,
476 - vec![chat_tool("search", None)],
477 - ToolStrictLevel::Function,
478 - );
479 -
480 - assert_eq!(tag_string(&pinned), tag_string(&strict));
481 - assert_ne!(tag_string(&pinned), tag_string(&envelope_only));
482 - }
483 -
484 - #[test]
485 - fn tool_strict_level_parses_case_insensitively() {
486 - assert_eq!("AUTO".parse::<ToolStrictLevel>(), Ok(ToolStrictLevel::Auto));
487 - assert_eq!(
488 - "PARAMETER".parse::<ToolStrictLevel>(),
489 - Ok(ToolStrictLevel::Parameter)
490 - );
491 - assert_eq!(
492 - "function".parse::<ToolStrictLevel>(),
493 - Ok(ToolStrictLevel::Function)
494 - );
495 - assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Auto);
496 - assert_eq!(
497 - serde_json::to_value(ToolStrictLevel::Auto).unwrap(),
498 - json!("auto")
499 - );
500 - assert_eq!(
501 - serde_json::from_value::<ToolStrictLevel>(json!("auto")).unwrap(),
502 - ToolStrictLevel::Auto
503 - );
504 - assert!("off".parse::<ToolStrictLevel>().is_err());
505 - assert!("strict".parse::<ToolStrictLevel>().is_err());
506 - }
507 -}
rust/src/chat/src/output/default/unified.rs
-20
@@ -48,25 +48,6 @@ impl UnifiedParserState {
48 }
49 }
50
51 - /// Initialize parser state once prompt token IDs are available.
52 - fn initialize(&mut self, prompt_token_ids: &[u32]) {
53 - if self.parser_failed {
54 - return;
55 - }
56 -
57 - match self.parser.initialize(prompt_token_ids) {
58 - Ok(()) => {}
59 - Err(error) => {
60 - warn!(
61 - error = %error.as_report(),
62 - "failed to initialize unified parser; falling back to plain text deltas"
63 - );
64 - self.parser_failed = true;
65 - self.open_call_index = None;
66 - }
67 - }
68 - }
69 -
51 /// Convert one decoded text delta into zero or more parsed assistant events.
52 fn process_delta(&mut self, delta: DecodedText) -> Result<Vec<AssistantEvent>> {
53 if self.parser_failed {
@@ -262,7 +243,6 @@ pub(crate) async fn unified_event_stream(
243 prompt_token_ids,
244 prompt_logprobs,
245 } => {
265 - state.initialize(&prompt_token_ids);
246 y.yield_ok(AssistantEvent::Start {
247 prompt_token_ids,
248 prompt_logprobs,
rust/src/chat/src/output/mod.rs
+20
@@ -7,6 +7,7 @@ use std::sync::Arc;
7 use futures::Stream;
8 use trait_set::trait_set;
9 use uuid::Uuid;
10 +use vllm_parser::output_grammar::BuiltOutputGrammar;
11 use vllm_text::output::{DecodedLogprobs, DecodedPromptLogprobs, DecodedTextEvent};
12
13 use crate::FinishReason;
@@ -15,11 +16,13 @@ use crate::event::{AssistantBlockKind, ChatEvent, ChatTokenUsage};
16
17 mod default;
18 mod harmony;
19 +mod structural_tag;
20 mod structured;
21
22 pub use default::DefaultChatOutputProcessor;
23 pub use harmony::HarmonyChatOutputProcessor;
24 pub(crate) use harmony::validate_harmony_parser_overrides;
25 +pub(crate) use structural_tag::apply_output_grammar;
26
27 /// Internal assistant event before final assembly.
28 ///
@@ -65,7 +68,24 @@ pub type DynChatEventStream = Pin<Box<dyn Stream<Item = Result<ChatEvent>> + Sen
68
69 /// Request-scoped output processor from decoded text events into structured
70 /// chat events.
71 +///
72 +/// A processor is created before the prompt is rendered, so that parser
73 +/// resolution can adjust the request, and then goes through two more phases
74 +/// once the final prompt token IDs are known: [`Self::initialize`] and
75 +/// [`Self::build_output_grammar`], in that order, both before
76 +/// [`Self::process`].
77 pub trait ChatOutputProcessor: Send {
78 + /// Initialize request-scoped parser state from the final prompt token IDs.
79 + fn initialize(&mut self, _prompt_token_ids: &[u32]) -> Result<()> {
80 + Ok(())
81 + }
82 +
83 + /// Build the structured output grammar implied by the initialized parser
84 + /// state, or `None` when this request needs no parser-owned grammar.
85 + fn build_output_grammar(&self) -> Result<Option<BuiltOutputGrammar>> {
86 + Ok(None)
87 + }
88 +
89 /// Consume decoded text stream and return the structured chat-event stream.
90 fn process(self: Box<Self>, decoded: DynDecodedTextEventStream) -> Result<DynChatEventStream>;
91 }
rust/src/chat/src/output/structural_tag.rs new
+73
@@ -0,0 +1,73 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +//! Applies parser-built xgrammar structural-tag constraints.
5 +
6 +use vllm_engine_core_client::protocol::structured_outputs::{
7 + StructuredOutputBackend, StructuredOutputsParams,
8 +};
9 +use vllm_parser::output_grammar::{BuiltOutputGrammar, GrammarCoverage};
10 +use vllm_text::TextRequest;
11 +use xgrammar_structural_tag::StructuralTag;
12 +
13 +use crate::{Error, Result};
14 +
15 +/// Apply one parser-built output grammar to the prepared text request.
16 +///
17 +/// A parser-owned grammar replaces any structured output constraint the user
18 +/// supplied, matching the Python frontend. `None` leaves the request untouched.
19 +pub(crate) fn apply_output_grammar(
20 + request: &mut TextRequest,
21 + built: Option<BuiltOutputGrammar>,
22 +) -> Result<()> {
23 + let Some(built) = built else {
24 + return Ok(());
25 + };
26 + let structural_tag = StructuralTag::new(built.format).to_json_string().map_err(|error| {
27 + Error::OutputGrammar {
28 + error: Box::new(error),
29 + }
30 + })?;
31 +
32 + // Overwrite any existing structured output settings with the structural tag constraint.
33 + request.sampling_params.structured_outputs = Some(StructuredOutputsParams {
34 + backend: StructuredOutputBackend::Xgrammar,
35 + ..StructuredOutputsParams::structural_tag(structural_tag)
36 + });
37 + request.reasoning_ended = match built.coverage {
38 + GrammarCoverage::FromTokenZero => Some(true),
39 + GrammarCoverage::FinalOutputOnly => None,
40 + };
41 +
42 + Ok(())
43 +}
44 +
45 +#[cfg(test)]
46 +mod tests {
47 + use serde_json::json;
48 + use vllm_engine_core_client::protocol::structured_outputs::{
49 + StructuredOutputBackend, StructuredOutputsParams,
50 + };
51 + use xgrammar_structural_tag::format::Format;
52 +
53 + use super::*;
54 +
55 + #[test]
56 + fn output_grammar_overwrites_answer_constraint() {
57 + let mut request = TextRequest::for_test();
58 + request.sampling_params.structured_outputs =
59 + Some(StructuredOutputsParams::json(json!({ "type": "object" })));
60 +
61 + let built = BuiltOutputGrammar::from_token_zero(Format::const_string("answer"));
62 + apply_output_grammar(&mut request, Some(built)).unwrap();
63 +
64 + assert_eq!(request.reasoning_ended, Some(true));
65 + let params = request.sampling_params.structured_outputs.unwrap();
66 + assert_eq!(params.backend, StructuredOutputBackend::Xgrammar);
67 + let serialized = params.constraint.as_structural_tag().unwrap();
68 + let value: serde_json::Value = serde_json::from_str(serialized).unwrap();
69 + assert_eq!(value["type"], "structural_tag");
70 + assert_eq!(value["format"]["type"], "const_string");
71 + assert_eq!(value["format"]["value"], "answer");
72 + }
73 +}
rust/src/chat/src/parser/mod.rs
+2 -60
@@ -12,6 +12,8 @@ use std::str::FromStr;
12
13 use serde_with::{DeserializeFromStr, SerializeDisplay};
14
15 +pub use vllm_parser::output_grammar::ToolStrictLevel;
16 +
17 use self::reasoning::ReasoningParserFactory;
18 use self::tool::ToolParserFactory;
19 use self::unified::UnifiedParserFactory;
@@ -57,66 +59,6 @@ impl ParserSelection {
59 }
60 }
61
60 -/// Server-side floor for tool-call structural tags (`--tool-strict-level`).
61 -#[derive(
62 - Debug,
63 - Clone,
64 - Copy,
65 - PartialEq,
66 - Eq,
67 - PartialOrd,
68 - Ord,
69 - Default,
70 - DeserializeFromStr,
71 - SerializeDisplay,
72 -)]
73 -pub enum ToolStrictLevel {
74 - /// Derive tool constraints from the request's tool choice and per-tool strictness.
75 - #[default]
76 - Auto,
77 - /// Constrain the tool-call envelope for every request with tools.
78 - Function,
79 - /// Additionally pin argument schemas, as if every tool were `strict: true`.
80 - Parameter,
81 -}
82 -
83 -impl ToolStrictLevel {
84 - pub const AUTO_LITERAL: &str = "auto";
85 - pub const FUNCTION_LITERAL: &str = "function";
86 - pub const PARAMETER_LITERAL: &str = "parameter";
87 -}
88 -
89 -impl FromStr for ToolStrictLevel {
90 - type Err = String;
91 -
92 - fn from_str(value: &str) -> Result<Self, Self::Err> {
93 - if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
94 - Ok(Self::Auto)
95 - } else if value.eq_ignore_ascii_case(Self::FUNCTION_LITERAL) {
96 - Ok(Self::Function)
97 - } else if value.eq_ignore_ascii_case(Self::PARAMETER_LITERAL) {
98 - Ok(Self::Parameter)
99 - } else {
100 - Err(format!(
101 - "unknown tool strict level {value:?}; expected one of {}, {}, {}",
102 - Self::AUTO_LITERAL,
103 - Self::FUNCTION_LITERAL,
104 - Self::PARAMETER_LITERAL
105 - ))
106 - }
107 - }
108 -}
109 -
110 -impl fmt::Display for ToolStrictLevel {
111 - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
112 - f.write_str(match self {
113 - Self::Auto => Self::AUTO_LITERAL,
114 - Self::Function => Self::FUNCTION_LITERAL,
115 - Self::Parameter => Self::PARAMETER_LITERAL,
116 - })
117 - }
118 -}
119 -
62 /// Validate explicit parser override names without starting request processing.
63 pub fn validate_parser_overrides(
64 tool_call_parser: &ParserSelection,
rust/src/chat/src/request.rs
+11
@@ -435,6 +435,17 @@ pub enum ChatToolChoice {
435 },
436 }
437
438 +impl From<&ChatToolChoice> for xgrammar_structural_tag::ToolChoice {
439 + fn from(tool_choice: &ChatToolChoice) -> Self {
440 + match tool_choice {
441 + ChatToolChoice::None => Self::none(),
442 + ChatToolChoice::Auto => Self::auto(),
443 + ChatToolChoice::Required => Self::required(),
444 + ChatToolChoice::Function { name } => Self::function(name.clone()),
445 + }
446 + }
447 +}
448 +
449 /// Resolved tool state shared by rendering, output parsing, and constraints.
450 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
451 pub struct ResolvedToolContext {
rust/src/chat/tests/roundtrip.rs
+2 -1
@@ -728,7 +728,7 @@ async fn parse_completion(
728 };
729
730 let mut request = base_request.clone();
731 - let processor = backends.chat_backend.new_chat_output_processor(
731 + let mut processor = backends.chat_backend.new_chat_output_processor(
732 &mut request,
733 NewChatOutputProcessorOptions {
734 tool_strict_level: vllm_chat::ToolStrictLevel::Auto,
@@ -736,6 +736,7 @@ async fn parse_completion(
736 reasoning_parser: &case.reasoning_parser,
737 },
738 )?;
739 + processor.initialize(&prompt_token_ids)?;
740
741 let decoded = decoded_completion_stream(
742 tokenizer.as_ref(),
rust/src/engine-core-client/src/protocol/request.rs
+1 -1
@@ -61,7 +61,7 @@ impl EngineCoreRequestType {
61 ///
62 /// Original Python construction point:
63 /// <https://github.com/vllm-project/vllm/blob/cec2ec11760f9f3beabd4c90451936078bf91533/vllm/entrypoints/openai/chat_completion/serving.py#L367-L369>
64 -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
64 +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
65 pub struct ReasoningParserKwargs {
66 /// Effective kwargs visible to the chat template for this request.
67 pub chat_template_kwargs: HashMap<String, serde_json::Value>,
rust/src/llm/examples/external_engine_smoke.rs
+1
@@ -62,6 +62,7 @@ fn build_request(request_id: String, max_tokens: u32) -> GenerateRequest {
62 session_id: None,
63 kv_hints: None,
64 reasoning_parser_kwargs: None,
65 + reasoning_ended: None,
66 lora_request: None,
67 }
68 }
rust/src/llm/src/request.rs
+8 -3
@@ -54,6 +54,11 @@ pub struct GenerateRequest {
54 /// Optional reasoning-parser kwargs forwarded to engine-side structured
55 /// output logic.
56 pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
57 + /// Optional engine reasoning-gate override.
58 + ///
59 + /// `Some(true)` means the submitted grammar covers reasoning from the first
60 + /// generated token.
61 + pub reasoning_ended: Option<bool>,
62 /// Optional LoRA adapter request applied to this generation.
63 pub lora_request: Option<LoraRequest>,
64 }
@@ -84,6 +89,7 @@ impl GenerateRequest {
89 session_id,
90 kv_hints,
91 reasoning_parser_kwargs,
92 + reasoning_ended,
93 lora_request,
94 } = self;
95
@@ -115,9 +121,7 @@ impl GenerateRequest {
121 session_id,
122 kv_hints,
123 external_req_id: Some(external_request_id),
118 - // Rust parser doesn't expose this information, leave it unset and let the
119 - // reasoning logic in engine-sided structured output manager handle it.
120 - reasoning_ended: None,
124 + reasoning_ended,
125 reasoning_parser_kwargs,
126 abort_immediately: false,
127 },
@@ -180,6 +184,7 @@ mod tests {
184 )]
185 .into(),
186 }),
187 + reasoning_ended: None,
188 lora_request: None,
189 }
190 }
rust/src/llm/tests/generate.rs
+1
@@ -169,6 +169,7 @@ fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest
169 session_id: None,
170 kv_hints: None,
171 reasoning_parser_kwargs: None,
172 + reasoning_ended: None,
173 lora_request: None,
174 }
175 }
rust/src/parser/Cargo.toml
+1
@@ -11,6 +11,7 @@ test-util = []
11 easy-ext.workspace = true
12 serde.workspace = true
13 serde_json.workspace = true
14 +serde_with.workspace = true
15 thiserror.workspace = true
16 thiserror-ext.workspace = true
17 vllm-tokenizer.workspace = true
rust/src/parser/benches/utils/adapter.rs
+1 -7
@@ -3,9 +3,7 @@
3
4 use std::sync::Arc;
5
6 -use vllm_parser::tool::{
7 - Result, StructuralTagBuilder, Tool, ToolParser, ToolParserError, ToolParserOutput,
8 -};
6 +use vllm_parser::tool::{Result, Tool, ToolParser, ToolParserError, ToolParserOutput};
7 use vllm_parser::unified::{
8 UnifiedParser, UnifiedParserError, UnifiedParserEvent, UnifiedParserOutput,
9 };
@@ -89,10 +87,6 @@ impl<T: UnifiedParser> ToolParser for UnifiedToolParserAdapter<T> {
87 self.inner.preserve_special_tokens()
88 }
89
92 - fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
93 - self.inner.structural_tag_builder()
94 - }
95 -
90 fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
91 self.inner.tool_call_id(tool_index)
92 }
rust/src/parser/src/lib.rs
+1
@@ -3,6 +3,7 @@
3
4 //! Streaming parsers for chat completions.
5
6 +pub mod output_grammar;
7 pub mod reasoning;
8 pub mod tool;
9 pub mod unified;
rust/src/parser/src/output_grammar.rs new
+362
@@ -0,0 +1,362 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +//! Output grammar construction shared by reasoning and tool parsers.
5 +
6 +use std::{fmt, str::FromStr};
7 +
8 +use serde_with::{DeserializeFromStr, SerializeDisplay};
9 +use thiserror::Error;
10 +use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagOptions};
11 +use xgrammar_structural_tag::format::Format;
12 +use xgrammar_structural_tag::tool::ToolChoiceValue;
13 +use xgrammar_structural_tag::{
14 + FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
15 +};
16 +
17 +use crate::tool::Tool;
18 +
19 +/// Result alias for output grammar construction.
20 +pub type Result<T> = std::result::Result<T, OutputGrammarError>;
21 +
22 +/// How much of the generated stream an output grammar covers.
23 +#[derive(Debug, Clone, Copy, PartialEq, Eq)]
24 +pub enum GrammarCoverage {
25 + /// The grammar covers reasoning and advances from the first generated token.
26 + FromTokenZero,
27 + /// The grammar covers only output emitted after reasoning ends.
28 + FinalOutputOnly,
29 +}
30 +
31 +/// One parser-built output grammar and its generated-stream coverage.
32 +#[derive(Debug, Clone, PartialEq)]
33 +pub struct BuiltOutputGrammar {
34 + /// Typed XGrammar structural-tag format.
35 + pub format: Format,
36 + /// Portion of the generated stream covered by `format`.
37 + pub coverage: GrammarCoverage,
38 +}
39 +
40 +impl BuiltOutputGrammar {
41 + /// A grammar that covers reasoning and everything after it.
42 + pub fn from_token_zero(format: Format) -> Self {
43 + Self {
44 + format,
45 + coverage: GrammarCoverage::FromTokenZero,
46 + }
47 + }
48 +
49 + /// A grammar that covers only the output emitted after reasoning ends.
50 + pub fn final_output_only(format: Format) -> Self {
51 + Self {
52 + format,
53 + coverage: GrammarCoverage::FinalOutputOnly,
54 + }
55 + }
56 +}
57 +
58 +/// Server-side floor for tool-call structural tags (`--tool-strict-level`).
59 +#[derive(
60 + Debug,
61 + Clone,
62 + Copy,
63 + PartialEq,
64 + Eq,
65 + PartialOrd,
66 + Ord,
67 + Default,
68 + DeserializeFromStr,
69 + SerializeDisplay,
70 +)]
71 +pub enum ToolStrictLevel {
72 + /// Derive tool constraints from the request's tool choice and per-tool strictness.
73 + #[default]
74 + Auto,
75 + /// Constrain the tool-call envelope for every request with tools.
76 + Function,
77 + /// Additionally pin argument schemas, as if every tool were `strict: true`.
78 + Parameter,
79 +}
80 +
81 +impl ToolStrictLevel {
82 + pub const AUTO_LITERAL: &str = "auto";
83 + pub const FUNCTION_LITERAL: &str = "function";
84 + pub const PARAMETER_LITERAL: &str = "parameter";
85 +}
86 +
87 +impl FromStr for ToolStrictLevel {
88 + type Err = String;
89 +
90 + fn from_str(value: &str) -> std::result::Result<Self, Self::Err> {
91 + if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) {
92 + Ok(Self::Auto)
93 + } else if value.eq_ignore_ascii_case(Self::FUNCTION_LITERAL) {
94 + Ok(Self::Function)
95 + } else if value.eq_ignore_ascii_case(Self::PARAMETER_LITERAL) {
96 + Ok(Self::Parameter)
97 + } else {
98 + Err(format!(
99 + "unknown tool strict level {value:?}; expected one of {}, {}, {}",
100 + Self::AUTO_LITERAL,
101 + Self::FUNCTION_LITERAL,
102 + Self::PARAMETER_LITERAL
103 + ))
104 + }
105 + }
106 +}
107 +
108 +impl fmt::Display for ToolStrictLevel {
109 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
110 + f.write_str(match self {
111 + Self::Auto => Self::AUTO_LITERAL,
112 + Self::Function => Self::FUNCTION_LITERAL,
113 + Self::Parameter => Self::PARAMETER_LITERAL,
114 + })
115 + }
116 +}
117 +
118 +/// Request facts available while an initialized parser builds an output grammar.
119 +pub struct OutputGrammarContext<'a> {
120 + /// Effective tools available for this generation.
121 + pub tools: &'a [Tool],
122 + /// Effective tool-choice policy.
123 + pub tool_choice: &'a ToolChoice,
124 + /// Server-side floor for tool-call constraints.
125 + pub tool_strict_level: ToolStrictLevel,
126 + /// Whether the request permits more than one tool call.
127 + pub parallel_tool_calls: bool,
128 +}
129 +
130 +/// Errors produced while building an output grammar.
131 +#[derive(Debug, Error)]
132 +pub enum OutputGrammarError {
133 + /// A model-specific structural-tag builder rejected its inputs.
134 + #[error("failed to build output grammar")]
135 + Build(#[from] xgrammar_structural_tag::Error),
136 +}
137 +
138 +/// Build the visible (post-reasoning) language from a crate structural-tag
139 +/// builder, or `None` when the request does not ask for strict tool calling.
140 +///
141 +/// This is the pre-existing strict-tool-calling grammar: the builder runs with
142 +/// `reasoning = false`, so the result describes only what follows the reasoning
143 +/// phase. It backs the default `ToolParser::build_visible_format` and the
144 +/// native unified parsers that have not yet grown their own builder.
145 +pub(crate) fn visible_format_from_builder(
146 + builder: Option<&dyn StructuralTagBuilder>,
147 + ctx: &OutputGrammarContext<'_>,
148 +) -> Result<Option<Format>> {
149 + let Some(builder) = builder else {
150 + return Ok(None);
151 + };
152 + if !tool_grammar_applies(ctx) {
153 + return Ok(None);
154 + }
155 +
156 + let structural_tag = build_structural_tag(
157 + builder,
158 + &tool_params(ctx.tools, ctx.tool_strict_level),
159 + ctx.tool_choice.clone(),
160 + StructuralTagOptions::default()
161 + .with_reasoning(false)
162 + .with_parallel_tool_calls(ctx.parallel_tool_calls),
163 + )?;
164 +
165 + Ok(Some(structural_tag.format))
166 +}
167 +
168 +fn tool_params(tools: &[Tool], strict_level: ToolStrictLevel) -> Vec<ToolParam> {
169 + tools
170 + .iter()
171 + .map(|tool| {
172 + ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
173 + name: tool.name.clone(),
174 + description: tool.description.clone(),
175 + parameters: Some(tool.parameters.clone()),
176 + strict: Some(
177 + strict_level >= ToolStrictLevel::Parameter || tool.strict == Some(true),
178 + ),
179 + }))
180 + })
181 + .collect()
182 +}
183 +
184 +/// Whether the request asks for a tool grammar at all.
185 +fn tool_grammar_applies(ctx: &OutputGrammarContext<'_>) -> bool {
186 + if ctx.tools.is_empty() {
187 + return false;
188 + }
189 +
190 + match ctx.tool_choice {
191 + ToolChoice::Value(ToolChoiceValue::None) => false,
192 + ToolChoice::Value(ToolChoiceValue::Auto) => {
193 + ctx.tool_strict_level >= ToolStrictLevel::Function
194 + || ctx.tools.iter().any(|tool| tool.strict == Some(true))
195 + }
196 + ToolChoice::Value(ToolChoiceValue::Required)
197 + | ToolChoice::NamedFunction(_)
198 + | ToolChoice::AllowedTools(_)
199 + | ToolChoice::FlatAllowedTools(_)
200 + | ToolChoice::Builtin(_) => true,
201 + }
202 +}
203 +
204 +#[cfg(test)]
205 +mod tests {
206 + use serde_json::json;
207 + use xgrammar_structural_tag::builders::Qwen3Builder;
208 +
209 + use super::*;
210 +
211 + fn tool(name: &str, strict: Option<bool>) -> Tool {
212 + Tool {
213 + name: name.to_string(),
214 + description: None,
215 + parameters: json!({
216 + "type": "object",
217 + "properties": { "query": { "type": "string" } },
218 + "required": ["query"]
219 + }),
220 + strict,
221 + }
222 + }
223 +
224 + fn build(tools: &[Tool], tool_choice: &ToolChoice) -> Option<Format> {
225 + visible_format_from_builder(
226 + Some(&Qwen3Builder),
227 + &OutputGrammarContext {
228 + tools,
229 + tool_choice,
230 + tool_strict_level: ToolStrictLevel::Auto,
231 + parallel_tool_calls: true,
232 + },
233 + )
234 + .unwrap()
235 + }
236 +
237 + #[test]
238 + fn tool_grammar_requires_tools_and_a_constraining_choice() {
239 + let non_strict = [tool("search", None)];
240 + assert!(build(&non_strict, &ToolChoice::none()).is_none());
241 + assert!(build(&non_strict, &ToolChoice::auto()).is_none());
242 + assert!(build(&non_strict, &ToolChoice::required()).is_some());
243 + assert!(build(&non_strict, &ToolChoice::function("search")).is_some());
244 +
245 + // `auto` only constrains once at least one tool is strict.
246 + let strict = [tool("search", Some(true)), tool("lookup", None)];
247 + assert!(build(&strict, &ToolChoice::auto()).is_some());
248 + }
249 +
250 + fn build_at_level(tool_choice: ToolChoice, tools: Vec<Tool>, level: ToolStrictLevel) -> Format {
251 + visible_format_from_builder(
252 + Some(&Qwen3Builder),
253 + &OutputGrammarContext {
254 + tools: &tools,
255 + tool_choice: &tool_choice,
256 + tool_strict_level: level,
257 + parallel_tool_calls: true,
258 + },
259 + )
260 + .unwrap()
261 + .unwrap()
262 + }
263 +
264 + fn tag_string(format: &Format) -> String {
265 + serde_json::to_string(format).unwrap()
266 + }
267 +
268 + #[test]
269 + fn function_level_constrains_auto_without_strict_tools() {
270 + let request = build_at_level(
271 + ToolChoice::auto(),
272 + vec![tool("search", None)],
273 + ToolStrictLevel::Function,
274 + );
275 +
276 + assert!(tag_string(&request).contains("search"));
277 + }
278 +
279 + #[test]
280 + fn absent_strict_is_non_strict() {
281 + let unset = build_at_level(
282 + ToolChoice::required(),
283 + vec![tool("search", None)],
284 + ToolStrictLevel::Auto,
285 + );
286 + let non_strict = build_at_level(
287 + ToolChoice::required(),
288 + vec![tool("search", Some(false))],
289 + ToolStrictLevel::Auto,
290 + );
291 + let strict = build_at_level(
292 + ToolChoice::required(),
293 + vec![tool("search", Some(true))],
294 + ToolStrictLevel::Auto,
295 + );
296 +
297 + assert_eq!(tag_string(&unset), tag_string(&non_strict));
298 + assert_ne!(tag_string(&unset), tag_string(&strict));
299 + }
300 +
301 + #[test]
302 + fn strict_tool_does_not_pin_its_neighbour() {
303 + let mixed = build_at_level(
304 + ToolChoice::auto(),
305 + vec![tool("weather", Some(true)), tool("search", None)],
306 + ToolStrictLevel::Auto,
307 + );
308 + let explicit = build_at_level(
309 + ToolChoice::auto(),
310 + vec![tool("weather", Some(true)), tool("search", Some(false))],
311 + ToolStrictLevel::Auto,
312 + );
313 +
314 + assert_eq!(tag_string(&mixed), tag_string(&explicit));
315 + }
316 +
317 + #[test]
318 + fn parameter_level_pins_argument_schemas() {
319 + let pinned = build_at_level(
320 + ToolChoice::auto(),
321 + vec![tool("search", None)],
322 + ToolStrictLevel::Parameter,
323 + );
324 + let strict = build_at_level(
325 + ToolChoice::auto(),
326 + vec![tool("search", Some(true))],
327 + ToolStrictLevel::Auto,
328 + );
329 + let envelope_only = build_at_level(
330 + ToolChoice::auto(),
331 + vec![tool("search", None)],
332 + ToolStrictLevel::Function,
333 + );
334 +
335 + assert_eq!(tag_string(&pinned), tag_string(&strict));
336 + assert_ne!(tag_string(&pinned), tag_string(&envelope_only));
337 + }
338 +
339 + #[test]
340 + fn tool_strict_level_parses_case_insensitively() {
341 + assert_eq!("AUTO".parse::<ToolStrictLevel>(), Ok(ToolStrictLevel::Auto));
342 + assert_eq!(
343 + "PARAMETER".parse::<ToolStrictLevel>(),
344 + Ok(ToolStrictLevel::Parameter)
345 + );
346 + assert_eq!(
347 + "function".parse::<ToolStrictLevel>(),
348 + Ok(ToolStrictLevel::Function)
349 + );
350 + assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Auto);
351 + assert_eq!(
352 + serde_json::to_value(ToolStrictLevel::Auto).unwrap(),
353 + json!("auto")
354 + );
355 + assert_eq!(
356 + serde_json::from_value::<ToolStrictLevel>(json!("auto")).unwrap(),
357 + ToolStrictLevel::Auto
358 + );
359 + assert!("off".parse::<ToolStrictLevel>().is_err());
360 + assert!("strict".parse::<ToolStrictLevel>().is_err());
361 + }
362 +}
rust/src/parser/src/reasoning/mod.rs
+12
@@ -31,6 +31,9 @@ mod step3p5;
31
32 use thiserror::Error;
33 use vllm_tokenizer::{DecodedText, DynTokenizer};
34 +use xgrammar_structural_tag::format::Format;
35 +
36 +use crate::output_grammar::{self, OutputGrammarContext};
37
38 pub use self::cohere_cmd::CohereCmdReasoningParser;
39 pub use self::deepseek_r1::DeepSeekR1ReasoningParser;
@@ -137,6 +140,15 @@ pub trait ReasoningParser: Send {
140 false
141 }
142
143 + /// Prepend the reasoning phase implied by the initialized parser state.
144 + fn wrap_visible_format(
145 + &self,
146 + _ctx: &OutputGrammarContext<'_>,
147 + _visible: &Format,
148 + ) -> output_grammar::Result<Option<Format>> {
149 + Ok(None)
150 + }
151 +
152 /// Feed one decoded text delta into the parser.
153 fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta>;
154
rust/src/parser/src/tool/mod.rs
+18
@@ -39,7 +39,9 @@ pub use seed_oss::SeedOssToolParser;
39 use serde::{Deserialize, Serialize};
40 use serde_json::Value;
41 pub use xgrammar_structural_tag::builders::StructuralTagBuilder;
42 +use xgrammar_structural_tag::format::Format;
43
44 +use crate::output_grammar::{self, OutputGrammarContext};
45 use crate::utils;
46
47 /// One function-style tool made available to the model.
@@ -189,7 +191,23 @@ pub trait ToolParser: Send {
191 false
192 }
193
194 + /// Build the language of everything the model may emit after reasoning
195 + /// ends: visible text plus tool calls. `None` means no tool grammar applies.
196 + ///
197 + /// The default forwards to [`Self::structural_tag_builder`] with reasoning
198 + /// disabled, which is exactly today's strict-tool-calling grammar.
199 + fn build_visible_format(
200 + &self,
201 + ctx: &OutputGrammarContext<'_>,
202 + ) -> output_grammar::Result<Option<Format>> {
203 + output_grammar::visible_format_from_builder(self.structural_tag_builder(), ctx)
204 + }
205 +
206 /// Return the xgrammar structural-tag builder used for strict tool calling.
207 + ///
208 + /// To be deprecated: this only exists to back the default
209 + /// [`Self::build_visible_format`]. New parsers should override that method
210 + /// directly instead of exposing a builder.
211 fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
212 None
213 }
rust/src/parser/src/unified/combined.rs
+25 -4
@@ -6,8 +6,9 @@
6 use vllm_tokenizer::{DecodedText, DynTokenizer};
7
8 use super::{Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput};
9 +use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
10 use crate::reasoning::ReasoningParser;
10 -use crate::tool::{StructuralTagBuilder, Tool, ToolParser, ToolParserOutput};
11 +use crate::tool::{Tool, ToolParser, ToolParserOutput};
12
13 /// Unified parser that composes existing reasoning and tool parsers.
14 pub struct CombinedParser {
@@ -79,8 +80,29 @@ impl UnifiedParser for CombinedParser {
80 || self.tool.as_ref().is_some_and(|parser| parser.preserve_special_tokens())
81 }
82
82 - fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
83 - self.tool.as_ref().and_then(|parser| parser.structural_tag_builder())
83 + fn build_output_grammar(
84 + &self,
85 + ctx: &OutputGrammarContext<'_>,
86 + ) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
87 + let Some(tool) = self.tool.as_ref() else {
88 + return Ok(None);
89 + };
90 + let Some(visible) = tool.build_visible_format(ctx)? else {
91 + return Ok(None);
92 + };
93 + // The reasoning parser may wrap that language with the reasoning phase
94 + // implied by its initialized (prompt-derived) state. If it does, the
95 + // grammar describes the stream from the first generated token and the
96 + // engine can skip its own reasoning gate; if it declines, the grammar
97 + // covers only the final output and the engine gate stays in charge.
98 + let wrapped = match self.reasoning.as_ref() {
99 + Some(reasoning) => reasoning.wrap_visible_format(ctx, &visible)?,
100 + None => None,
101 + };
102 + Ok(Some(match wrapped {
103 + Some(full) => BuiltOutputGrammar::from_token_zero(full),
104 + None => BuiltOutputGrammar::final_output_only(visible),
105 + }))
106 }
107
108 fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
@@ -332,7 +354,6 @@ mod tests {
354 fn combined_parser_emits_tool_calls_from_visible_content() {
355 let tool = Qwen3XmlToolParser::create(&test_tools()).unwrap();
356 let mut parser = CombinedParser::new(None, Some(tool));
335 - assert!(parser.structural_tag_builder().is_some());
357
358 let output = collect(
359 &mut parser,
rust/src/parser/src/unified/hy/hy_v3.rs
+22 -25
@@ -4,8 +4,9 @@
4 use vllm_tokenizer::{DecodedText, DynTokenizer};
5
6 use super::detect_hy_token_suffix;
7 +use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
8 use crate::reasoning::HyReasoningParser;
8 -use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, StructuralTagBuilder, Tool};
9 +use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, Tool};
10 use crate::unified::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id};
11
12 /// Unified reasoning and tool parser for HY3 output.
@@ -46,8 +47,11 @@ impl UnifiedParser for HyV3UnifiedParser {
47 self.inner.preserve_special_tokens()
48 }
49
49 - fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
50 - self.inner.structural_tag_builder()
50 + fn build_output_grammar(
51 + &self,
52 + ctx: &OutputGrammarContext<'_>,
53 + ) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
54 + self.inner.build_output_grammar(ctx)
55 }
56
57 fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
@@ -73,12 +77,10 @@ mod tests {
77
78 use serde_json::json;
79 use vllm_tokenizer::{DecodedText, Tokenizer, test_utils::TestTokenizer};
76 - use xgrammar_structural_tag::builders::StructuralTagOptions;
77 - use xgrammar_structural_tag::{
78 - FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
79 - };
80 + use xgrammar_structural_tag::{StructuralTag, ToolChoice};
81
82 use super::{HyV3UnifiedParser, UnifiedParser};
83 + use crate::output_grammar::{GrammarCoverage, OutputGrammarContext};
84 use crate::tool::Tool;
85 use crate::unified::{UnifiedParserEvent, UnifiedParserOutput};
86
@@ -151,24 +153,19 @@ mod tests {
153
154 #[test]
155 fn structural_tag_uses_tokenizer_detected_suffix() {
154 - let parser = HyV3UnifiedParser::new(&tools(), Arc::new(tokenizer())).unwrap();
155 - let structural_tools = [ToolParam::Function(FunctionToolParam::new(
156 - FunctionDefinition::new("get_weather").with_parameters(json!({
157 - "type": "object",
158 - "properties": { "city": { "type": "string" } },
159 - "required": ["city"]
160 - })),
161 - ))];
162 -
163 - let tag = build_structural_tag(
164 - parser.structural_tag_builder().unwrap(),
165 - &structural_tools,
166 - ToolChoice::required(),
167 - StructuralTagOptions::default().with_reasoning(false),
168 - )
169 - .unwrap()
170 - .to_json_string()
171 - .unwrap();
156 + let tools = tools();
157 + let parser = HyV3UnifiedParser::new(&tools, Arc::new(tokenizer())).unwrap();
158 + let tag = parser
159 + .build_output_grammar(&OutputGrammarContext {
160 + tools: &tools,
161 + tool_strict_level: Default::default(),
162 + parallel_tool_calls: true,
163 + tool_choice: &ToolChoice::required(),
164 + })
165 + .unwrap()
166 + .unwrap();
167 + assert_eq!(tag.coverage, GrammarCoverage::FinalOutputOnly);
168 + let tag = StructuralTag::new(tag.format).to_json_string().unwrap();
169
170 assert!(tag.contains("<tool_calls:opensource>"));
171 assert!(tag.contains("<arg_value:opensource>"));
rust/src/parser/src/unified/hy/hy_v4.rs
+22 -25
@@ -4,8 +4,9 @@
4 use vllm_tokenizer::{DecodedText, DynTokenizer};
5
6 use super::detect_hy_token_suffix;
7 +use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
8 use crate::reasoning::HyReasoningParser;
8 -use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, StructuralTagBuilder, Tool};
9 +use crate::tool::{HyDialect, HyToolMarkers, HyToolParser, Tool};
10 use crate::unified::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id};
11
12 /// Unified reasoning and tool parser for HY4 output.
@@ -46,8 +47,11 @@ impl UnifiedParser for HyV4UnifiedParser {
47 self.inner.preserve_special_tokens()
48 }
49
49 - fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
50 - self.inner.structural_tag_builder()
50 + fn build_output_grammar(
51 + &self,
52 + ctx: &OutputGrammarContext<'_>,
53 + ) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
54 + self.inner.build_output_grammar(ctx)
55 }
56
57 fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
@@ -73,12 +77,10 @@ mod tests {
77
78 use serde_json::json;
79 use vllm_tokenizer::{DecodedText, Tokenizer, test_utils::TestTokenizer};
76 - use xgrammar_structural_tag::builders::StructuralTagOptions;
77 - use xgrammar_structural_tag::{
78 - FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
79 - };
80 + use xgrammar_structural_tag::{StructuralTag, ToolChoice};
81
82 use super::{HyV4UnifiedParser, UnifiedParser};
83 + use crate::output_grammar::{GrammarCoverage, OutputGrammarContext};
84 use crate::tool::Tool;
85 use crate::unified::{UnifiedParserEvent, UnifiedParserOutput};
86
@@ -164,24 +166,19 @@ mod tests {
166
167 #[test]
168 fn structural_tag_uses_compact_hy4_skeleton() {
167 - let parser = HyV4UnifiedParser::new(&tools(), Arc::new(tokenizer())).unwrap();
168 - let structural_tools = [ToolParam::Function(FunctionToolParam::new(
169 - FunctionDefinition::new("get_weather").with_parameters(json!({
170 - "type": "object",
171 - "properties": { "city": { "type": "string" } },
172 - "required": ["city"]
173 - })),
174 - ))];
175 -
176 - let tag = build_structural_tag(
177 - parser.structural_tag_builder().unwrap(),
178 - &structural_tools,
179 - ToolChoice::required(),
180 - StructuralTagOptions::default().with_reasoning(false),
181 - )
182 - .unwrap()
183 - .to_json_string()
184 - .unwrap();
169 + let tools = tools();
170 + let parser = HyV4UnifiedParser::new(&tools, Arc::new(tokenizer())).unwrap();
171 + let tag = parser
172 + .build_output_grammar(&OutputGrammarContext {
173 + tools: &tools,
174 + tool_strict_level: Default::default(),
175 + parallel_tool_calls: true,
176 + tool_choice: &ToolChoice::required(),
177 + })
178 + .unwrap()
179 + .unwrap();
180 + assert_eq!(tag.coverage, GrammarCoverage::FinalOutputOnly);
181 + let tag = StructuralTag::new(tag.format).to_json_string().unwrap();
182
183 assert!(tag.contains("<tool_calls:opensource>"));
184 assert!(tag.contains("<tool_call:opensource>get_weather"));
rust/src/parser/src/unified/kimi_k3.rs
+12 -3
@@ -51,7 +51,10 @@ use winnow::token::{literal, rest, take_till, take_until, take_while};
51
52 use self::structural_tag::KIMI_K3_STRUCTURAL_TAG_BUILDER;
53 use super::{Result, UnifiedParser, UnifiedParserOutput, token_id};
54 -use crate::tool::{StructuralTagBuilder, Tool, ToolCallDelta};
54 +use crate::output_grammar::{
55 + self, BuiltOutputGrammar, OutputGrammarContext, visible_format_from_builder,
56 +};
57 +use crate::tool::{Tool, ToolCallDelta};
58 use crate::unified::parsing_failed;
59 use crate::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker};
60
@@ -275,8 +278,14 @@ impl UnifiedParser for KimiK3UnifiedParser {
278 true
279 }
280
278 - fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
279 - Some(&KIMI_K3_STRUCTURAL_TAG_BUILDER)
281 + fn build_output_grammar(
282 + &self,
283 + ctx: &OutputGrammarContext<'_>,
284 + ) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
285 + Ok(
286 + visible_format_from_builder(Some(&KIMI_K3_STRUCTURAL_TAG_BUILDER), ctx)?
287 + .map(BuiltOutputGrammar::final_output_only),
288 + )
289 }
290
291 fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> {
rust/src/parser/src/unified/mod.rs
+8 -6
@@ -18,10 +18,9 @@ use thiserror::Error;
18 use thiserror_ext::Macro;
19 use vllm_tokenizer::{DecodedText, DynTokenizer};
20
21 +use crate::output_grammar::{self, BuiltOutputGrammar, OutputGrammarContext};
22 use crate::reasoning::ReasoningError;
22 -use crate::tool::{
23 - StructuralTagBuilder, Tool, ToolCallDelta, ToolParserError, ToolParserEvent, ToolParserOutput,
24 -};
23 +use crate::tool::{Tool, ToolCallDelta, ToolParserError, ToolParserEvent, ToolParserOutput};
24
25 /// Result alias for unified parser operations.
26 pub type Result<T> = std::result::Result<T, UnifiedParserError>;
@@ -240,9 +239,12 @@ pub trait UnifiedParser: Send {
239 false
240 }
241
243 - /// Return the xgrammar structural-tag builder used for strict tool calling.
244 - fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
245 - None
242 + /// Build the request output grammar after prompt-based initialization.
243 + fn build_output_grammar(
244 + &self,
245 + _ctx: &OutputGrammarContext<'_>,
246 + ) -> output_grammar::Result<Option<BuiltOutputGrammar>> {
247 + Ok(None)
248 }
249
250 /// Return the parser-provided ID for a tool call by index, if the model emitted one.
rust/src/server/src/grpc/convert.rs
+2 -1
@@ -137,7 +137,8 @@ pub fn to_text_request(
137 data_parallel_rank: None,
138 session_id,
139 kv_hints,
140 - reasoning_parser_kwargs: None,
140 + reasoning_parser_kwargs: Default::default(),
141 + reasoning_ended: None,
142 lora_request: None,
143 arrival_time: None,
144 })
rust/src/server/src/routes/inference/generate/convert.rs
+2 -1
@@ -83,7 +83,8 @@ pub(super) fn prepare_generate_request(
83 data_parallel_rank: ctx.data_parallel_rank,
84 session_id: ctx.session_id,
85 kv_hints: None,
86 - reasoning_parser_kwargs: None,
86 + reasoning_parser_kwargs: Default::default(),
87 + reasoning_ended: None,
88 lora_request: lora_resolution.lora_request.clone(),
89 arrival_time: None,
90 };
rust/src/server/src/routes/openai/completions/convert.rs
+2 -1
@@ -156,7 +156,8 @@ pub(super) fn prepare_completion_request(
156 data_parallel_rank: ctx.data_parallel_rank,
157 session_id,
158 kv_hints: None,
159 - reasoning_parser_kwargs: None,
159 + reasoning_parser_kwargs: Default::default(),
160 + reasoning_ended: None,
161 lora_request: lora_resolution.lora_request.clone(),
162 arrival_time: None,
163 };
rust/src/server/src/routes/render.rs
+1 -1
@@ -124,7 +124,7 @@ async fn render_chat(
124 let chat_request = lower_chat_request(body, &model_resolution(&state), request_context)?;
125 let (text_request, _) = state
126 .chat
127 - .prepare(chat_request)
127 + .prepare(chat_request, &state.text)
128 .await
129 .map_err(|error| ApiError::invalid_request(error.to_report_string(), None))?;
130 Ok(Json(lower_render_request(
rust/src/server/src/routes/tokenize/types.rs
+2 -1
@@ -55,7 +55,8 @@ impl TokenizeCompletionRequest {
55 data_parallel_rank: None,
56 session_id: None,
57 kv_hints: None,
58 - reasoning_parser_kwargs: None,
58 + reasoning_parser_kwargs: Default::default(),
59 + reasoning_ended: None,
60 lora_request: None,
61 arrival_time: None,
62 }
rust/src/text/src/lib.rs
+21
@@ -104,6 +104,27 @@ impl TextRequestProcessor {
104 Ok(prompt_token_ids)
105 }
106
107 + /// Tokenize and truncate the prompt in place, so that later preparation
108 + /// reuses the final token IDs instead of tokenizing again.
109 + ///
110 + /// Frontends that need the final prompt tokens before submission (for
111 + /// example, to initialize a prompt-aware output parser) call this first
112 + /// and then hand the same request to [`TextLlm::generate`].
113 + pub fn tokenize_in_place<'r>(&self, request: &'r mut TextRequest) -> Result<&'r [u32]> {
114 + request.validate()?;
115 + let prompt_token_ids = self.prepare_prompt_tokens(
116 + take(&mut request.prompt),
117 + request.add_special_tokens,
118 + request.prompt_truncation.take(),
119 + request.sampling_params.max_tokens,
120 + )?;
121 + request.prompt = Prompt::TokenIds(prompt_token_ids);
122 + match &request.prompt {
123 + Prompt::TokenIds(prompt_token_ids) => Ok(prompt_token_ids),
124 + Prompt::Text(_) => unreachable!("prompt was just replaced with token IDs"),
125 + }
126 + }
127 +
128 /// Prepare one request's prompt tokens without generation-specific lowering.
129 pub fn tokenize(&self, request: TextRequest) -> Result<Vec<u32>> {
130 request.validate()?;
rust/src/text/src/lower.rs
+2 -1
@@ -61,7 +61,8 @@ pub fn lower_text_request(
61 data_parallel_rank: request.data_parallel_rank,
62 session_id: request.session_id.clone(),
63 kv_hints: request.kv_hints.clone(),
64 - reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),
64 + reasoning_parser_kwargs: Some(request.reasoning_parser_kwargs.clone()),
65 + reasoning_ended: request.reasoning_ended,
66 lora_request: request.lora_request.clone(),
67 arrival_time: request.arrival_time,
68 trace_headers: None,
rust/src/text/src/request.rs
+14 -4
@@ -215,10 +215,19 @@ pub struct TextRequest {
215 /// Optional orchestrator-originated KV hints.
216 #[serde(default)]
217 pub kv_hints: Option<KvHintsEnvelope>,
218 - /// Optional reasoning-parser kwargs forwarded to engine-side structured
219 - /// output logic.
218 + /// Reasoning-parser kwargs forwarded to engine-side structured output
219 + /// logic. The engine consults them only when it owns grammar activation;
220 + /// see [`Self::reasoning_ended`].
221 #[serde(default)]
221 - pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,
222 + pub reasoning_parser_kwargs: ReasoningParserKwargs,
223 + /// Optional engine reasoning-gate override selected by a higher-level frontend.
224 + ///
225 + /// `Some(true)` means the structured output grammar covers reasoning from
226 + /// the first generated token, so the engine masks and advances immediately
227 + /// instead of waiting for its reasoning parser. Unset for final-output-only
228 + /// grammars and for requests without a grammar.
229 + #[serde(default)]
230 + pub reasoning_ended: Option<bool>,
231 /// LoRA adapter selected for this request.
232 #[serde(default)]
233 pub lora_request: Option<LoraRequest>,
@@ -247,7 +256,8 @@ impl TextRequest {
256 data_parallel_rank: None,
257 session_id: None,
258 kv_hints: None,
250 - reasoning_parser_kwargs: None,
259 + reasoning_parser_kwargs: Default::default(),
260 + reasoning_ended: None,
261 lora_request: None,
262 arrival_time: None,
263 }