@tduncan / Nvidia-Optimization-Stack-VLLM / commits / fc6b6e1feb

[Rust Frontend] Correctly parse whitespace framing in model output (#55417)

Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Codex <noreply@openai.com>

Bugen Zhao committed Sep 9, 2026 at 13:40 UTC fc6b6e1feb263c385be634566cef89dd4dcf1292
36 files changed +781 -316
rust/src/chat/src/lib.rs
+1 -1
@@ -416,6 +416,6 @@ mod tests {
416 )
417 .unwrap_err();
418
419 - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
419 + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
420 }
421 }
rust/src/chat/src/parser/reasoning/mod.rs
+8 -6
@@ -7,10 +7,10 @@ use std::sync::{Arc, LazyLock};
7
8 pub use vllm_parser::reasoning::{
9 CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DeepSeekV3ReasoningParser,
10 - DeepSeekV4ReasoningParser, Glm45ReasoningParser, KimiK2ReasoningParser, KimiReasoningParser,
11 - MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser, NemotronV3ReasoningParser,
12 - Qwen3ReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, SeedOssReasoningParser,
13 - Step3ReasoningParser, Step3p5ReasoningParser,
10 + DeepSeekV4ReasoningParser, Glm45ReasoningParser, Glm47ReasoningParser, KimiK2ReasoningParser,
11 + KimiReasoningParser, MiniMaxM2ReasoningParser, MiniMaxM3ReasoningParser,
12 + NemotronV3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta, ReasoningError,
13 + ReasoningParser, SeedOssReasoningParser, Step3ReasoningParser, Step3p5ReasoningParser,
14 };
15 use vllm_tokenizer::DynTokenizer;
16
@@ -25,6 +25,7 @@ pub mod names {
25 pub const GEMMA4: &str = "gemma4";
26 pub const INKLING: &str = "inkling";
27 pub const GLM45: &str = "glm45";
28 + pub const GLM47: &str = "glm47";
29 pub const HY_V3: &str = "hy_v3";
30 pub const HY_V4: &str = "hy_v4";
31 pub const KIMI: &str = "kimi";
@@ -69,6 +70,7 @@ impl ReasoningParserFactory {
70 .register_unified_dummy(names::GEMMA4)
71 .register_unified_dummy(names::INKLING)
72 .register_parser::<Glm45ReasoningParser>(names::GLM45)
73 + .register_parser::<Glm47ReasoningParser>(names::GLM47)
74 .register_unified_dummy(names::HY_V3)
75 .register_unified_dummy(names::HY_V4)
76 .register_parser::<KimiReasoningParser>(names::KIMI)
@@ -91,8 +93,8 @@ impl ReasoningParserFactory {
93 .register_pattern("gemma4", names::GEMMA4)
94 .register_pattern("qwq", names::DEEPSEEK_R1)
95 .register_pattern("qwen3", names::QWEN3)
94 - .register_pattern("glm-5", names::GLM45)
95 - .register_pattern("glm-4.7", names::GLM45)
96 + .register_pattern("glm-5", names::GLM47)
97 + .register_pattern("glm-4.7", names::GLM47)
98 .register_pattern("glm-4.6", names::GLM45)
99 .register_pattern("glm-4.5", names::GLM45)
100 .register_pattern("hy3", names::HY_V3)
rust/src/chat/src/parser/reasoning/tests.rs
+12 -1
@@ -25,7 +25,7 @@ fn factory_contains_and_lists_registered_parsers() {
25 }
26
27 #[test]
28 -fn factory_resolves_deepseek_v4_to_qwen3_alias() {
28 +fn factory_resolves_deepseek_v4() {
29 let factory = ReasoningParserFactory::new();
30 assert_eq!(
31 factory.resolve_name_for_model("deepseek-ai/DeepSeek-V4"),
@@ -112,3 +112,14 @@ fn factory_rejects_unknown_parser_names() {
112 };
113 assert!(error.to_string().contains("choose from"));
114 }
115 +
116 +#[test]
117 +fn factory_distinguishes_glm_reasoning_framing() {
118 + let factory = ReasoningParserFactory::new();
119 + for model in ["zai-org/GLM-4.5", "zai-org/GLM-4.6"] {
120 + assert_eq!(factory.resolve_name_for_model(model), Some(names::GLM45));
121 + }
122 + for model in ["zai-org/GLM-4.7-Flash", "zai-org/GLM-5.2-FP8"] {
123 + assert_eq!(factory.resolve_name_for_model(model), Some(names::GLM47));
124 + }
125 +}
rust/src/chat/tests/roundtrip.rs
+11 -11
@@ -445,10 +445,12 @@ async fn run_roundtrip_reasoning_and_content_inner(
445 let result = run_roundtrip(case, backends, &request, assistant).await?;
446
447 assert_eq!(
448 - result.parsed_message.reasoning().as_deref().map(str::trim),
449 - effective_thinking.then_some(expected_reasoning)
448 + result.parsed_message.reasoning().as_deref(),
449 + effective_thinking.then_some(expected_reasoning),
450 + "parsed message: {:#?}",
451 + result.parsed_message
452 );
451 - assert_eq!(result.parsed_message.text().trim(), expected_text);
453 + assert_eq!(result.parsed_message.text(), expected_text);
454 assert_eq!(result.parsed_message.tool_calls().count(), 0);
455
456 assert_eq!(
@@ -508,10 +510,12 @@ async fn run_roundtrip_tool_call_mix(
510 .await?;
511
512 assert_eq!(
511 - result.parsed_message.reasoning().as_deref().map(str::trim),
512 - Some(expected_reasoning)
513 + result.parsed_message.reasoning().as_deref(),
514 + Some(expected_reasoning),
515 + "parsed message: {:#?}",
516 + result.parsed_message
517 );
514 - assert_eq!(result.parsed_message.text().trim(), expected_text);
518 + assert_eq!(result.parsed_message.text(), expected_text);
519
520 let tool_calls = result.parsed_message.tool_calls().collect::<Vec<_>>();
521 assert_eq!(
@@ -714,11 +718,7 @@ async fn parse_completion(
718
719 while let Some(event) = events.next().await {
720 if let ChatEvent::Done { message, .. } = event? {
717 - // TODO: currently our parsers are not very strict about preserving or trimming
718 - // whitespace, so we trim here to avoid roundtrip failures due to
719 - // insignificant whitespace differences. However, this may hurt token-level
720 - // fidelity so we should consider improving them.
721 - return Ok(message.trim());
721 + return Ok(message);
722 }
723 }
724
rust/src/parser/src/reasoning/cohere_cmd.rs
+8 -5
@@ -3,7 +3,10 @@
3
4 use vllm_tokenizer::{DecodedText, DynTokenizer};
5
6 -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
6 +use super::{
7 + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
8 + Result,
9 +};
10
11 /// Reasoning parser for Cohere Command models that use explicit START/END tags.
12 pub struct CohereCmdReasoningParser {
@@ -15,11 +18,12 @@ impl CohereCmdReasoningParser {
18 /// machine.
19 pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
20 Ok(Self {
18 - inner: DelimitedReasoningParser::new(
21 + inner: DelimitedReasoningParserBuilder::new(
22 tokenizer,
23 "<|START_THINKING|>",
24 "<|END_THINKING|>",
22 - )?,
25 + )
26 + .build()?,
27 })
28 }
29 }
@@ -33,8 +37,7 @@ impl ReasoningParser for CohereCmdReasoningParser {
37 }
38
39 fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
36 - self.inner.initialize(prompt_token_ids);
37 - Ok(())
40 + self.inner.initialize(prompt_token_ids)
41 }
42
43 fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
rust/src/parser/src/reasoning/deepseek_r1.rs
+9 -6
@@ -3,7 +3,10 @@
3
4 use vllm_tokenizer::{DecodedText, DynTokenizer};
5
6 -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
6 +use super::{
7 + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
8 + Result,
9 +};
10
11 /// Reasoning parser for DeepSeek R1 style outputs.
12 pub struct DeepSeekR1ReasoningParser {
@@ -11,11 +14,12 @@ pub struct DeepSeekR1ReasoningParser {
14 }
15
16 impl DeepSeekR1ReasoningParser {
14 - /// Create a DeepSeek R1 parser backed by the shared delimited state
15 - /// machine.
17 + /// Create a DeepSeek R1 parser backed by the shared delimited state machine.
18 pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
19 Ok(Self {
18 - inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>")?,
20 + inner: DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
21 + .with_after_start("\n")
22 + .build()?,
23 })
24 }
25 }
@@ -29,8 +33,7 @@ impl ReasoningParser for DeepSeekR1ReasoningParser {
33 }
34
35 fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
32 - self.inner.initialize(prompt_token_ids);
33 - Ok(())
36 + self.inner.initialize(prompt_token_ids)
37 }
38
39 fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
rust/src/parser/src/reasoning/deepseek_v3.rs new
+45
@@ -0,0 +1,45 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +use vllm_tokenizer::{DecodedText, DynTokenizer};
5 +
6 +use super::{
7 + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
8 + Result,
9 +};
10 +
11 +/// Reasoning parser for DeepSeek V3 style outputs.
12 +pub struct DeepSeekV3ReasoningParser {
13 + inner: DelimitedReasoningParser,
14 +}
15 +
16 +impl DeepSeekV3ReasoningParser {
17 + /// Create a DeepSeek V3 parser backed by the shared delimited state machine.
18 + pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
19 + Ok(Self {
20 + inner: DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
21 + .build()?,
22 + })
23 + }
24 +}
25 +
26 +impl ReasoningParser for DeepSeekV3ReasoningParser {
27 + fn create(tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
28 + where
29 + Self: Sized + 'static,
30 + {
31 + Ok(Box::new(Self::new(tokenizer)?))
32 + }
33 +
34 + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
35 + self.inner.initialize(prompt_token_ids)
36 + }
37 +
38 + fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
39 + Ok(self.inner.push(delta))
40 + }
41 +
42 + fn finish(&mut self) -> Result<ReasoningDelta> {
43 + Ok(self.inner.finish())
44 + }
45 +}
rust/src/parser/src/reasoning/delimited.rs
+174 -99
@@ -4,6 +4,90 @@
4 use vllm_tokenizer::{DecodedText, DynTokenizer, Tokenizer};
5
6 use super::{ReasoningDelta, ReasoningError, Result};
7 +use crate::utils::partial_prefix_len;
8 +
9 +/// Build a delimited reasoning parser with fixed text at each boundary.
10 +pub(crate) struct DelimitedReasoningParserBuilder {
11 + tokenizer: DynTokenizer,
12 + start_token: String,
13 + end_token: String,
14 + before_start: &'static str,
15 + after_start: &'static str,
16 + before_end: &'static str,
17 + after_end: &'static str,
18 +}
19 +
20 +impl DelimitedReasoningParserBuilder {
21 + /// Configure the tokenizer and reasoning delimiters.
22 + pub(crate) fn new(
23 + tokenizer: DynTokenizer,
24 + start_token: impl Into<String>,
25 + end_token: impl Into<String>,
26 + ) -> Self {
27 + Self {
28 + tokenizer,
29 + start_token: start_token.into(),
30 + end_token: end_token.into(),
31 + before_start: "",
32 + after_start: "",
33 + before_end: "",
34 + after_end: "",
35 + }
36 + }
37 +
38 + /// Include a fixed prefix when it immediately precedes the start marker.
39 + pub(crate) fn with_before_start(mut self, before_start: &'static str) -> Self {
40 + self.before_start = before_start;
41 + self
42 + }
43 +
44 + /// Consume fixed text immediately after the start marker.
45 + pub(crate) fn with_after_start(mut self, after_start: &'static str) -> Self {
46 + self.after_start = after_start;
47 + self
48 + }
49 +
50 + /// Include a fixed prefix when it immediately precedes the end marker.
51 + pub(crate) fn with_before_end(mut self, before_end: &'static str) -> Self {
52 + self.before_end = before_end;
53 + self
54 + }
55 +
56 + /// Consume fixed text immediately after the end marker.
57 + pub(crate) fn with_after_end(mut self, after_end: &'static str) -> Self {
58 + self.after_end = after_end;
59 + self
60 + }
61 +
62 + /// Create one delimited parser state machine.
63 + pub(crate) fn build(self) -> Result<DelimitedReasoningParser> {
64 + let start_token_id = self.tokenizer.token_to_id(&self.start_token).ok_or_else(|| {
65 + ReasoningError::MissingToken {
66 + token: self.start_token.clone(),
67 + }
68 + })?;
69 + let end_token_id = self.tokenizer.token_to_id(&self.end_token).ok_or_else(|| {
70 + ReasoningError::MissingToken {
71 + token: self.end_token.clone(),
72 + }
73 + })?;
74 +
75 + Ok(DelimitedReasoningParser {
76 + tokenizer: self.tokenizer,
77 + framed_start_token: format!("{}{}", self.before_start, self.start_token),
78 + start_token: self.start_token,
79 + framed_end_token: format!("{}{}", self.before_end, self.end_token),
80 + end_token: self.end_token,
81 + start_token_id,
82 + end_token_id,
83 + after_start: self.after_start,
84 + after_end: self.after_end,
85 + current_in_reasoning: false,
86 + buffer: DecodedText::default(),
87 + pending_after: "",
88 + })
89 + }
90 +}
91
92 /// Shared incremental state machine for tag-delimited reasoning protocols.
93 ///
@@ -18,54 +102,45 @@ use super::{ReasoningDelta, ReasoningError, Result};
102 /// prefill different prompts.
103 pub(crate) struct DelimitedReasoningParser {
104 tokenizer: DynTokenizer,
21 - current_in_reasoning: bool,
22 - buffer: DecodedText,
105 start_token: String,
106 + framed_start_token: String,
107 end_token: String,
108 start_token_id: u32,
109 end_token_id: u32,
110 + after_start: &'static str,
111 + after_end: &'static str,
112 + framed_end_token: String,
113 +
114 + // Mutable state.
115 + current_in_reasoning: bool,
116 + buffer: DecodedText,
117 + pending_after: &'static str,
118 }
119
120 impl DelimitedReasoningParser {
30 - /// Create one delimited parser state machine.
31 - pub(crate) fn new(
32 - tokenizer: DynTokenizer,
33 - start_token: impl Into<String>,
34 - end_token: impl Into<String>,
35 - ) -> Result<Self> {
36 - let start_token = start_token.into();
37 - let end_token = end_token.into();
38 - let start_token_id =
39 - tokenizer
40 - .token_to_id(&start_token)
41 - .ok_or_else(|| ReasoningError::MissingToken {
42 - token: start_token.clone(),
43 - })?;
44 - let end_token_id =
45 - tokenizer.token_to_id(&end_token).ok_or_else(|| ReasoningError::MissingToken {
46 - token: end_token.clone(),
47 - })?;
48 -
49 - Ok(Self {
50 - tokenizer,
51 - current_in_reasoning: false,
52 - buffer: DecodedText::default(),
53 - start_token,
54 - end_token,
55 - start_token_id,
56 - end_token_id,
57 - })
58 - }
59 -
60 - /// Initialize the starting state from prompt token IDs.
61 - pub(crate) fn initialize(&mut self, prompt_token_ids: &[u32]) {
62 - self.current_in_reasoning = last_reasoning_boundary(
121 + /// Initialize the starting state and remaining framing from prompt tokens.
122 + pub(crate) fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
123 + self.buffer = DecodedText::default();
124 + self.current_in_reasoning = false;
125 + self.pending_after = "";
126 + if let Some((index, in_reasoning)) = last_reasoning_boundary_position(
127 prompt_token_ids,
128 self.start_token_id,
129 self.end_token_id,
130 self.tokenizer.as_ref(),
67 - )
68 - .unwrap_or(false);
131 + ) {
132 + self.current_in_reasoning = in_reasoning;
133 + self.pending_after = if in_reasoning {
134 + self.after_start
135 + } else {
136 + self.after_end
137 + };
138 + if !self.pending_after.is_empty() && index + 1 < prompt_token_ids.len() {
139 + let tail = self.tokenizer.decode(&prompt_token_ids[index + 1..], false)?;
140 + self.pending_after = self.pending_after.strip_prefix(&tail).unwrap_or("");
141 + }
142 + }
143 + Ok(())
144 }
145
146 /// Return whether the parser is currently inside a reasoning section.
@@ -76,84 +151,74 @@ impl DelimitedReasoningParser {
151 /// Parse one decoded text delta and return its reasoning/content split.
152 pub(crate) fn push(&mut self, delta: DecodedText) -> ReasoningDelta {
153 self.buffer.append(delta);
79 -
80 - let partial_suffix_len = self.partial_suffix_len(&self.buffer.text);
81 - let stable_len = self.buffer.text.len() - partial_suffix_len;
82 - let stable = self.buffer.drain_prefix(stable_len);
83 -
84 - self.parse_stable_text(stable)
154 + self.parse_buffer(false)
155 }
156
87 - /// Flush any buffered partial delimiter suffix at end of stream.
157 + /// Flush partial framing and delimiters as body text at end of stream.
158 pub(crate) fn finish(&mut self) -> ReasoningDelta {
89 - // `drain_prefix(text.len())` takes trailing zero-width tokens too.
90 - let stable = self.buffer.drain_prefix(self.buffer.text.len());
91 - self.parse_stable_text(stable)
159 + self.parse_buffer(true)
160 }
161
94 - /// Parse text that is known not to end with a partial delimiter suffix.
95 - ///
162 /// Reasoning and content pieces keep the attributions of the tokens that
163 /// produced them; delimiter marker spans are drained and dropped, keeping
164 /// marker tokens out of any count.
99 - fn parse_stable_text(&mut self, mut stable: DecodedText) -> ReasoningDelta {
165 + fn parse_buffer(&mut self, finishing: bool) -> ReasoningDelta {
166 let mut delta = ReasoningDelta::default();
101 -
102 - while !stable.text.is_empty() {
103 - if self.current_in_reasoning {
104 - if let Some(end_idx) = stable.text.find(&self.end_token) {
105 - delta.push_reasoning(stable.drain_prefix(end_idx));
106 - let _ = stable.drain_prefix(self.end_token.len());
107 - self.current_in_reasoning = false;
108 - } else {
109 - delta.push_reasoning(stable);
110 - return delta;
167 + loop {
168 + if !self.pending_after.is_empty() {
169 + if self.buffer.text.starts_with(self.pending_after) {
170 + let _ = self.buffer.drain_prefix(self.pending_after.len());
171 + } else if !finishing && self.pending_after.starts_with(&self.buffer.text) {
172 + break;
173 }
112 - } else if let Some(start_idx) = stable.text.find(&self.start_token) {
113 - delta.push_content(stable.drain_prefix(start_idx));
114 - let _ = stable.drain_prefix(self.start_token.len());
115 - self.current_in_reasoning = true;
116 - } else {
117 - delta.push_content(stable);
118 - return delta;
174 + // A mismatch preserves the candidate as body text.
175 + self.pending_after = "";
176 }
120 - }
177
122 - // A remainder with empty text may still carry zero-width tokens;
123 - // attribute them to the current state.
124 - if !stable.attributions.is_empty() {
125 - if self.current_in_reasoning {
126 - delta.push_reasoning(stable);
178 + let (marker, framed_marker) = if self.current_in_reasoning {
179 + (&self.end_token, &self.framed_end_token)
180 } else {
128 - delta.push_content(stable);
181 + (&self.start_token, &self.framed_start_token)
182 + };
183 + if let Some(index) = self.buffer.text.find(marker) {
184 + let before = &framed_marker[..framed_marker.len() - marker.len()];
185 + let body_len = if self.buffer.text[..index].ends_with(before) {
186 + index - before.len()
187 + } else {
188 + index
189 + };
190 + let boundary_len = index + marker.len() - body_len;
191 + let body = self.buffer.drain_prefix(body_len);
192 + self.push_body(&mut delta, body);
193 + let _ = self.buffer.drain_prefix(boundary_len);
194 + self.current_in_reasoning = !self.current_in_reasoning;
195 + self.pending_after = if self.current_in_reasoning {
196 + self.after_start
197 + } else {
198 + self.after_end
199 + };
200 + continue;
201 }
130 - }
202
203 + let keep_len = if finishing {
204 + 0
205 + } else {
206 + partial_prefix_len(&self.buffer.text, marker)
207 + .max(partial_prefix_len(&self.buffer.text, framed_marker))
208 + };
209 + let body = self.buffer.drain_prefix(self.buffer.text.len() - keep_len);
210 + self.push_body(&mut delta, body);
211 + break;
212 + }
213 delta
214 }
215
135 - /// Return the longest trailing suffix that could still complete a
136 - /// delimiter.
137 - fn partial_suffix_len(&self, text: &str) -> usize {
138 - let mut best = 0;
139 - for idx in text.char_indices().map(|(idx, _)| idx).skip(1) {
140 - let suffix = &text[idx..];
141 - if self.start_token.starts_with(suffix) && self.start_token != suffix {
142 - best = best.max(text.len() - idx);
143 - }
144 - if self.end_token.starts_with(suffix) && self.end_token != suffix {
145 - best = best.max(text.len() - idx);
146 - }
216 + fn push_body(&self, delta: &mut ReasoningDelta, body: DecodedText) {
217 + if self.current_in_reasoning {
218 + delta.push_reasoning(body);
219 + } else {
220 + delta.push_content(body);
221 }
148 -
149 - if self.start_token.starts_with(text) && self.start_token != text {
150 - best = best.max(text.len());
151 - }
152 - if self.end_token.starts_with(text) && self.end_token != text {
153 - best = best.max(text.len());
154 - }
155 -
156 - best
222 }
223 }
224
@@ -164,12 +229,22 @@ pub(crate) fn last_reasoning_boundary(
229 end_token_id: u32,
230 tokenizer: &dyn Tokenizer,
231 ) -> Option<bool> {
167 - for token_id in prompt_token_ids.iter().rev().copied() {
232 + last_reasoning_boundary_position(prompt_token_ids, start_token_id, end_token_id, tokenizer)
233 + .map(|(_, in_reasoning)| in_reasoning)
234 +}
235 +
236 +fn last_reasoning_boundary_position(
237 + prompt_token_ids: &[u32],
238 + start_token_id: u32,
239 + end_token_id: u32,
240 + tokenizer: &dyn Tokenizer,
241 +) -> Option<(usize, bool)> {
242 + for (index, &token_id) in prompt_token_ids.iter().enumerate().rev() {
243 if token_id == start_token_id {
169 - return Some(true);
244 + return Some((index, true));
245 }
246 if token_id == end_token_id {
172 - return Some(false);
247 + return Some((index, false));
248 }
249 if tokenizer.is_special_id(token_id) {
250 return None;
rust/src/parser/src/reasoning/glm45.rs new
+48
@@ -0,0 +1,48 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +use vllm_tokenizer::{DecodedText, DynTokenizer};
5 +
6 +use super::{
7 + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
8 + Result,
9 +};
10 +
11 +/// Reasoning parser for GLM-4.5/4.6 models using `<think>`/`</think>`
12 +/// delimiters.
13 +pub struct Glm45ReasoningParser {
14 + inner: DelimitedReasoningParser,
15 +}
16 +
17 +impl Glm45ReasoningParser {
18 + /// Create a GLM-4.5/4.6 parser backed by the shared delimited state machine.
19 + pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
20 + Ok(Self {
21 + inner: DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
22 + .with_before_start("\n")
23 + .with_after_end("\n")
24 + .build()?,
25 + })
26 + }
27 +}
28 +
29 +impl ReasoningParser for Glm45ReasoningParser {
30 + fn create(tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
31 + where
32 + Self: Sized + 'static,
33 + {
34 + Ok(Box::new(Self::new(tokenizer)?))
35 + }
36 +
37 + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
38 + self.inner.initialize(prompt_token_ids)
39 + }
40 +
41 + fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
42 + Ok(self.inner.push(delta))
43 + }
44 +
45 + fn finish(&mut self) -> Result<ReasoningDelta> {
46 + Ok(self.inner.finish())
47 + }
48 +}
rust/src/parser/src/reasoning/hy.rs
+8 -5
@@ -3,7 +3,10 @@
3
4 use vllm_tokenizer::{DecodedText, DynTokenizer};
5
6 -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, Result};
6 +use super::{
7 + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningError,
8 + ReasoningParser, Result,
9 +};
10
11 /// Internal HY reasoning stage used by the unified HY parsers.
12 pub(crate) struct HyReasoningParser {
@@ -14,11 +17,12 @@ impl HyReasoningParser {
17 /// Create a HY reasoning parser for the tokenizer-specific marker suffix.
18 pub(crate) fn new(tokenizer: DynTokenizer, suffix: &str) -> Result<Self> {
19 Ok(Self {
17 - inner: DelimitedReasoningParser::new(
20 + inner: DelimitedReasoningParserBuilder::new(
21 tokenizer,
22 format!("<think{suffix}>"),
23 format!("</think{suffix}>"),
21 - )?,
24 + )
25 + .build()?,
26 })
27 }
28 }
@@ -36,8 +40,7 @@ impl ReasoningParser for HyReasoningParser {
40 }
41
42 fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
39 - self.inner.initialize(prompt_token_ids);
40 - Ok(())
43 + self.inner.initialize(prompt_token_ids)
44 }
45
46 fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
rust/src/parser/src/reasoning/kimi.rs
+7 -4
@@ -3,7 +3,10 @@
3
4 use vllm_tokenizer::{DecodedText, DynTokenizer};
5
6 -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
6 +use super::{
7 + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
8 + Result,
9 +};
10
11 /// Reasoning parser for legacy Kimi models that use Unicode thinking tags.
12 pub struct KimiReasoningParser {
@@ -14,7 +17,8 @@ impl KimiReasoningParser {
17 /// Create a Kimi parser backed by the shared delimited state machine.
18 pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
19 Ok(Self {
17 - inner: DelimitedReasoningParser::new(tokenizer, "◁think▷", "◁/think▷")?,
20 + inner: DelimitedReasoningParserBuilder::new(tokenizer, "◁think▷", "◁/think▷")
21 + .build()?,
22 })
23 }
24 }
@@ -28,8 +32,7 @@ impl ReasoningParser for KimiReasoningParser {
32 }
33
34 fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
31 - self.inner.initialize(prompt_token_ids);
32 - Ok(())
35 + self.inner.initialize(prompt_token_ids)
36 }
37
38 fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
rust/src/parser/src/reasoning/minimax_m3.rs
+7 -3
@@ -3,7 +3,10 @@
3
4 use vllm_tokenizer::{DecodedText, DynTokenizer};
5
6 -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
6 +use super::{
7 + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
8 + Result,
9 +};
10
11 const M3_THINK_START: &str = "<mm:think>";
12 const M3_THINK_END: &str = "</mm:think>";
@@ -27,7 +30,8 @@ impl MiniMaxM3ReasoningParser {
30 /// Create a MiniMax M3 parser backed by the shared delimited state machine.
31 pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
32 Ok(Self {
30 - inner: DelimitedReasoningParser::new(tokenizer, M3_THINK_START, M3_THINK_END)?,
33 + inner: DelimitedReasoningParserBuilder::new(tokenizer, M3_THINK_START, M3_THINK_END)
34 + .build()?,
35 at_response_start: true,
36 leading_end_buffer: DecodedText::default(),
37 })
@@ -77,7 +81,7 @@ impl ReasoningParser for MiniMaxM3ReasoningParser {
81 }
82
83 fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
80 - self.inner.initialize(prompt_token_ids);
84 + self.inner.initialize(prompt_token_ids)?;
85 self.at_response_start = true;
86 self.leading_end_buffer.clear();
87 Ok(())
rust/src/parser/src/reasoning/mod.rs
+21 -14
@@ -19,7 +19,9 @@
19
20 mod cohere_cmd;
21 mod deepseek_r1;
22 +mod deepseek_v3;
23 mod delimited;
24 +mod glm45;
25 mod hy;
26 mod kimi;
27 mod minimax_m3;
@@ -32,7 +34,11 @@ use vllm_tokenizer::{DecodedText, DynTokenizer};
34
35 pub use self::cohere_cmd::CohereCmdReasoningParser;
36 pub use self::deepseek_r1::DeepSeekR1ReasoningParser;
35 -pub(crate) use self::delimited::{DelimitedReasoningParser, last_reasoning_boundary};
37 +pub use self::deepseek_v3::DeepSeekV3ReasoningParser;
38 +pub(crate) use self::delimited::{
39 + DelimitedReasoningParser, DelimitedReasoningParserBuilder, last_reasoning_boundary,
40 +};
41 +pub use self::glm45::Glm45ReasoningParser;
42 pub(crate) use self::hy::HyReasoningParser;
43 pub use self::kimi::KimiReasoningParser;
44 pub use self::minimax_m3::MiniMaxM3ReasoningParser;
@@ -40,22 +46,21 @@ pub use self::qwen3::Qwen3ReasoningParser;
46 pub use self::seed_oss::SeedOssReasoningParser;
47 pub use self::step3p5::Step3p5ReasoningParser;
48
43 -/// DeepSeek V3 currently shares the standard `<think>...</think>` parser.
44 -pub type DeepSeekV3ReasoningParser = Qwen3ReasoningParser;
45 -/// DeepSeek V4 currently shares the standard `<think>...</think>` parser.
46 -pub type DeepSeekV4ReasoningParser = Qwen3ReasoningParser;
47 -/// GLM45 currently shares the standard `<think>...</think>` parser.
48 -pub type Glm45ReasoningParser = Qwen3ReasoningParser;
49 -/// Kimi K2 currently shares the standard `<think>...</think>` parser.
49 +/// Nemotron V3 and Step3p5 frame all three reasoning boundaries with one newline.
50 +pub type NemotronV3ReasoningParser = Step3p5ReasoningParser;
51 +/// Step3 and DeepSeek R1 prefill a newline after the reasoning opener.
52 +pub type Step3ReasoningParser = DeepSeekR1ReasoningParser;
53 +
54 +/// DeepSeek V4 currently shares the DeepSeek V3 reasoning parser.
55 +pub type DeepSeekV4ReasoningParser = DeepSeekV3ReasoningParser;
56 +/// GLM-4.7 and GLM-5 use bare reasoning delimiters.
57 +pub type Glm47ReasoningParser = DeepSeekV3ReasoningParser;
58 +/// Kimi K2 currently shares the DeepSeek V3 reasoning parser.
59 // TODO: kimi k2 may implicitly end reasoning by starting a tool call section
60 // using <|tool_calls_section_begin|>, we should support that.
52 -pub type KimiK2ReasoningParser = Qwen3ReasoningParser;
53 -/// MiniMax M2 currently shares the standard `<think>...</think>` parser.
61 +pub type KimiK2ReasoningParser = DeepSeekV3ReasoningParser;
62 +/// MiniMax M2 currently shares the Qwen3 reasoning parser.
63 pub type MiniMaxM2ReasoningParser = Qwen3ReasoningParser;
55 -/// Nemotron V3 currently shares the standard `<think>...</think>` parser.
56 -pub type NemotronV3ReasoningParser = Qwen3ReasoningParser;
57 -/// Step3 currently shares the standard `<think>...</think>` parser.
58 -pub type Step3ReasoningParser = Qwen3ReasoningParser;
64
65 /// Result alias for reasoning parser operations.
66 pub type Result<T> = std::result::Result<T, ReasoningError>;
@@ -140,6 +145,8 @@ pub trait ReasoningParser: Send {
145 /// Errors produced while creating or running reasoning parsers.
146 #[derive(Debug, Error)]
147 pub enum ReasoningError {
148 + #[error(transparent)]
149 + Tokenizer(#[from] vllm_tokenizer::TokenizerError),
150 #[error("tokenizer is missing reasoning delimiter token `{token}`")]
151 MissingToken { token: String },
152 #[error(
rust/src/parser/src/reasoning/qwen3.rs
+11 -7
@@ -3,13 +3,14 @@
3
4 use vllm_tokenizer::{DecodedText, DynTokenizer};
5
6 -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
6 +use super::{
7 + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
8 + Result,
9 +};
10
11 /// Reasoning parser for the Qwen3/Qwen3.5 family.
12 ///
10 -/// This parser uses standard `<think>...</think>` delimiters and defaults to
11 -/// waiting for an explicit start token when prompt initialization finds no
12 -/// reasoning boundary.
13 +/// This parser uses standard `<think>...</think>` delimiters.
14 pub struct Qwen3ReasoningParser {
15 inner: DelimitedReasoningParser,
16 }
@@ -18,7 +19,11 @@ impl Qwen3ReasoningParser {
19 /// Create a Qwen3 parser backed by the shared delimited state machine.
20 pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
21 Ok(Self {
21 - inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>")?,
22 + inner: DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
23 + .with_after_start("\n")
24 + .with_before_end("\n")
25 + .with_after_end("\n\n")
26 + .build()?,
27 })
28 }
29 }
@@ -32,8 +37,7 @@ impl ReasoningParser for Qwen3ReasoningParser {
37 }
38
39 fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
35 - self.inner.initialize(prompt_token_ids);
36 - Ok(())
40 + self.inner.initialize(prompt_token_ids)
41 }
42
43 fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
rust/src/parser/src/reasoning/seed_oss.rs
+8 -4
@@ -3,7 +3,10 @@
3
4 use vllm_tokenizer::{DecodedText, DynTokenizer};
5
6 -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
6 +use super::{
7 + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
8 + Result,
9 +};
10
11 /// Reasoning parser for SeedOSS models using `<seed:think>`/`</seed:think>`
12 /// delimiters.
@@ -15,7 +18,9 @@ impl SeedOssReasoningParser {
18 /// Create a SeedOSS parser backed by the shared delimited state machine.
19 pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
20 Ok(Self {
18 - inner: DelimitedReasoningParser::new(tokenizer, "<seed:think>", "</seed:think>")?,
21 + inner: DelimitedReasoningParserBuilder::new(tokenizer, "<seed:think>", "</seed:think>")
22 + .with_after_end("\n")
23 + .build()?,
24 })
25 }
26 }
@@ -29,8 +34,7 @@ impl ReasoningParser for SeedOssReasoningParser {
34 }
35
36 fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
32 - self.inner.initialize(prompt_token_ids);
33 - Ok(())
37 + self.inner.initialize(prompt_token_ids)
38 }
39
40 fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
rust/src/parser/src/reasoning/step3p5.rs
+15 -94
@@ -3,89 +3,27 @@
3
4 use vllm_tokenizer::{DecodedText, DynTokenizer};
5
6 -use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningParser, Result};
7 -
8 -/// Reasoning parser for Step3p5 outputs.
9 -///
10 -/// Step3p5 uses standard `<think>`/`</think>` delimiters but emits a `\n`
11 -/// immediately before and/or after `</think>`. The parser drops these framing
12 -/// newlines on both sides of the boundary, holding a trailing `\n` from
13 -/// reasoning across pushes until either more reasoning text or `</think>`
14 -/// arrives, and dropping a leading `\n` from the first content delta after
15 -/// the boundary.
6 +use super::{
7 + DelimitedReasoningParser, DelimitedReasoningParserBuilder, ReasoningDelta, ReasoningParser,
8 + Result,
9 +};
10 +
11 +/// Reasoning parser for Step3p5 style outputs.
12 pub struct Step3p5ReasoningParser {
13 inner: DelimitedReasoningParser,
18 - /// `\n` at end of last reasoning delta, held in case `</think>` follows.
19 - pending_reasoning_newline: Option<DecodedText>,
20 - /// Last push ended on `</think>` without emitting content; the next
21 - /// content delta's leading `\n` should be dropped.
22 - just_ended_reasoning: bool,
14 }
15
16 impl Step3p5ReasoningParser {
17 /// Create a Step3p5 parser backed by the shared delimited state machine.
18 pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
19 Ok(Self {
29 - inner: DelimitedReasoningParser::new(tokenizer, "<think>", "</think>")?,
30 - pending_reasoning_newline: None,
31 - just_ended_reasoning: false,
20 + inner: DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
21 + .with_after_start("\n")
22 + .with_before_end("\n")
23 + .with_after_end("\n")
24 + .build()?,
25 })
26 }
34 -
35 - /// Drop framing newlines around `</think>` and track held-newline state.
36 - fn process(
37 - &mut self,
38 - mut inner_delta: ReasoningDelta,
39 - was_in_reasoning: bool,
40 - now_in_reasoning: bool,
41 - ) -> ReasoningDelta {
42 - // A `<think>...</think>` round-trip in one push still counts as a
43 - // transition: the inner emits reasoning while ending in content mode.
44 - let transitioned =
45 - !now_in_reasoning && (was_in_reasoning || inner_delta.reasoning.is_some());
46 -
47 - // Replay or drop a previously-held trailing reasoning newline.
48 - if self.pending_reasoning_newline.is_some() {
49 - if let Some(reasoning) = inner_delta.reasoning.as_mut() {
50 - let mut held = self.pending_reasoning_newline.take().unwrap();
51 - held.append(std::mem::take(reasoning));
52 - *reasoning = held;
53 - } else if transitioned {
54 - // The held `\n` was the one right before `</think>`: drop it.
55 - self.pending_reasoning_newline = None;
56 - }
57 - }
58 -
59 - // Hold back a trailing reasoning `\n` until we know if `</think>` follows.
60 - if let Some(reasoning) = inner_delta.reasoning.as_mut()
61 - && reasoning.text.ends_with('\n')
62 - {
63 - let kept = reasoning.drain_prefix(reasoning.text.len() - 1);
64 - let newline = std::mem::replace(reasoning, kept);
65 - if !transitioned {
66 - self.pending_reasoning_newline = Some(newline);
67 - }
68 - }
69 -
70 - // Drop a leading `\n` of content emitted right after `</think>`.
71 - if let Some(content) = inner_delta.content.as_mut()
72 - && (transitioned || self.just_ended_reasoning)
73 - && content.text.starts_with('\n')
74 - {
75 - let _ = content.drain_prefix(1);
76 - }
77 -
78 - self.just_ended_reasoning = transitioned && inner_delta.content.is_none();
79 -
80 - if inner_delta.reasoning.as_ref().is_some_and(DecodedText::is_empty) {
81 - inner_delta.reasoning = None;
82 - }
83 - if inner_delta.content.as_ref().is_some_and(DecodedText::is_empty) {
84 - inner_delta.content = None;
85 - }
86 -
87 - inner_delta
88 - }
27 }
28
29 impl ReasoningParser for Step3p5ReasoningParser {
@@ -97,32 +35,15 @@ impl ReasoningParser for Step3p5ReasoningParser {
35 }
36
37 fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
100 - self.inner.initialize(prompt_token_ids);
101 - Ok(())
38 + self.inner.initialize(prompt_token_ids)
39 }
40
41 fn push(&mut self, delta: DecodedText) -> Result<ReasoningDelta> {
105 - let was = self.inner.in_reasoning();
106 - let inner_delta = self.inner.push(delta);
107 - let now = self.inner.in_reasoning();
108 - Ok(self.process(inner_delta, was, now))
42 + Ok(self.inner.push(delta))
43 }
44
45 fn finish(&mut self) -> Result<ReasoningDelta> {
112 - let was = self.inner.in_reasoning();
113 - let inner_delta = self.inner.finish();
114 - let now = self.inner.in_reasoning();
115 - let mut delta = self.process(inner_delta, was, now);
116 -
117 - // Emit a still-held newline rather than silently dropping it.
118 - if let Some(held) = self.pending_reasoning_newline.take() {
119 - match delta.reasoning.as_mut() {
120 - Some(existing) => existing.append(held),
121 - None => delta.reasoning = Some(held),
122 - }
123 - }
124 -
125 - Ok(delta)
46 + Ok(self.inner.finish())
47 }
48 }
49
@@ -181,7 +102,7 @@ mod tests {
102 // `</think>`; surrounding newlines remain part of reasoning/content.
103 let tokenizer = Arc::new(fake_tokenizer());
104 let mut parser = Step3p5ReasoningParser::new(tokenizer).unwrap();
184 - parser.initialize(&[THINK_START_ID]).unwrap();
105 + parser.initialize(&[THINK_START_ID, u32::from(b'\n')]).unwrap();
106
107 let delta = push_str(
108 &mut parser,
rust/src/parser/src/reasoning/tests.rs
+190 -10
@@ -8,8 +8,9 @@ use vllm_tokenizer::{DecodedText, DynTokenizer, TokenAnchor, TokenAttribution};
8
9 use super::{
10 CohereCmdReasoningParser, DeepSeekR1ReasoningParser, DelimitedReasoningParser,
11 - KimiReasoningParser, MiniMaxM3ReasoningParser, Qwen3ReasoningParser, ReasoningDelta,
12 - ReasoningParser, Result, SeedOssReasoningParser, Step3p5ReasoningParser,
11 + DelimitedReasoningParserBuilder, KimiReasoningParser, MiniMaxM3ReasoningParser,
12 + Qwen3ReasoningParser, ReasoningDelta, ReasoningParser, Result, SeedOssReasoningParser,
13 + Step3p5ReasoningParser,
14 };
15
16 pub(crate) const THINK_START_ID: u32 = 256;
@@ -57,7 +58,9 @@ pub(crate) fn content_str(delta: &ReasoningDelta) -> Option<&str> {
58 #[test]
59 fn delimited_content_only_stream() {
60 let tokenizer = Arc::new(fake_tokenizer());
60 - let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
61 + let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
62 + .build()
63 + .unwrap();
64
65 let delta = parser.push(DecodedText::unattributed("plain content"));
66 assert_eq!(content_str(&delta), Some("plain content"));
@@ -66,7 +69,9 @@ fn delimited_content_only_stream() {
69 #[test]
70 fn delimited_single_chunk_with_reasoning_and_content() {
71 let tokenizer = Arc::new(fake_tokenizer());
69 - let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
72 + let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
73 + .build()
74 + .unwrap();
75
76 let delta = parser.push(DecodedText::unattributed("<think>reason</think>answer"));
77 assert_eq!(reasoning_str(&delta), Some("reason"));
@@ -76,7 +81,9 @@ fn delimited_single_chunk_with_reasoning_and_content() {
81 #[test]
82 fn delimited_partial_tokens_across_chunks() {
83 let tokenizer = Arc::new(fake_tokenizer());
79 - let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
84 + let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
85 + .build()
86 + .unwrap();
87
88 assert!(parser.push(DecodedText::unattributed("<thi")).is_empty());
89 let delta = parser.push(DecodedText::unattributed("nk>reason</think>answer"));
@@ -87,8 +94,10 @@ fn delimited_partial_tokens_across_chunks() {
94 #[test]
95 fn delimited_finish_flushes_buffer() {
96 let tokenizer = Arc::new(fake_tokenizer());
90 - let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
91 - parser.initialize(&[THINK_START_ID]);
97 + let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
98 + .build()
99 + .unwrap();
100 + parser.initialize(&[THINK_START_ID]).unwrap();
101
102 let delta = parser.push(DecodedText::unattributed("unfinished</thi"));
103 assert_eq!(reasoning_str(&delta), Some("unfinished"));
@@ -99,8 +108,10 @@ fn delimited_finish_flushes_buffer() {
108 #[test]
109 fn delimited_zero_width_only_piece_is_attributed_to_current_state() {
110 let tokenizer = Arc::new(fake_tokenizer());
102 - let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
103 - parser.initialize(&[THINK_START_ID]);
111 + let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
112 + .build()
113 + .unwrap();
114 + parser.initialize(&[THINK_START_ID]).unwrap();
115
116 // A filtered special token produces a zero-width attribution with no text;
117 // it must survive as a reasoning piece rather than being dropped.
@@ -130,7 +141,9 @@ fn delimited_zero_width_only_piece_is_attributed_to_current_state() {
141 #[test]
142 fn delimited_marker_tokens_are_dropped_from_attributions() {
143 let tokenizer = Arc::new(fake_tokenizer());
133 - let mut parser = DelimitedReasoningParser::new(tokenizer, "<think>", "</think>").unwrap();
144 + let mut parser = DelimitedReasoningParserBuilder::new(tokenizer, "<think>", "</think>")
145 + .build()
146 + .unwrap();
147
148 let mut collected = CollectedAttributions::default();
149 for chunk in attributed_chunks(&[
@@ -283,3 +296,170 @@ pub(crate) fn collect_attributed(
296 collected.record(parser.finish().unwrap());
297 collected
298 }
299 +
300 +fn framed_parser() -> DelimitedReasoningParser {
301 + DelimitedReasoningParserBuilder::new(Arc::new(fake_tokenizer()), "<think>", "</think>")
302 + .with_after_start("\n")
303 + .with_before_end("\n")
304 + .with_after_end("\n\n")
305 + .build()
306 + .unwrap()
307 +}
308 +
309 +fn collect_framed(parser: &mut DelimitedReasoningParser, chunks: &[&str]) -> (String, String) {
310 + let mut result = (String::new(), String::new());
311 + for chunk in chunks.iter().copied().map(Some).chain(std::iter::once(None)) {
312 + let delta = match chunk {
313 + Some(chunk) => parser.push(DecodedText::unattributed(chunk)),
314 + None => parser.finish(),
315 + };
316 + if let Some(reasoning) = delta.reasoning {
317 + result.0.push_str(&reasoning.text);
318 + }
319 + if let Some(content) = delta.content {
320 + result.1.push_str(&content.text);
321 + }
322 + }
323 + result
324 +}
325 +
326 +#[test]
327 +fn delimited_framing_preserves_body_whitespace_at_every_split() {
328 + let wire = "<think>\n\n reason\t\n\n</think>\n\n\n answer\n";
329 + let expected = ("\n reason\t\n".to_string(), "\n answer\n".to_string());
330 + for split in 0..=wire.len() {
331 + let mut parser = framed_parser();
332 + assert_eq!(
333 + collect_framed(&mut parser, &[&wire[..split], &wire[split..]]),
334 + expected,
335 + "split at {split}"
336 + );
337 + }
338 + let chars = wire
339 + .as_bytes()
340 + .chunks(1)
341 + .map(|c| std::str::from_utf8(c).unwrap())
342 + .collect::<Vec<_>>();
343 + assert_eq!(collect_framed(&mut framed_parser(), &chars), expected);
344 +}
345 +
346 +#[test]
347 +fn delimited_partial_framing_is_preserved_on_mismatch_or_finish() {
348 + for (wire, reasoning, content) in [
349 + ("<think>reason</think>\nanswer", "reason", "\nanswer"),
350 + ("<think>\nreason\nmore\n", "reason\nmore\n", ""),
351 + ("<think>\nreason\n</thi", "reason\n</thi", ""),
352 + ("<think>\nreason\n</think>\n", "reason", "\n"),
353 + (" plain\n", "", " plain\n"),
354 + ] {
355 + for split in 0..=wire.len() {
356 + assert_eq!(
357 + collect_framed(&mut framed_parser(), &[&wire[..split], &wire[split..]]),
358 + (reasoning.to_string(), content.to_string()),
359 + "wire {wire:?}, split {split}"
360 + );
361 + }
362 + }
363 +}
364 +
365 +#[test]
366 +fn delimited_prompt_initialization_consumes_only_remaining_framing() {
367 + use vllm_tokenizer::Tokenizer;
368 + let tokenizer = fake_tokenizer();
369 + for (prompt, wire, reasoning, content) in [
370 + (
371 + "<think>",
372 + "\nreason\n</think>\n\nanswer",
373 + "reason",
374 + "answer",
375 + ),
376 + (
377 + "<think>\n",
378 + "\nreason\n</think>\n\nanswer",
379 + "\nreason",
380 + "answer",
381 + ),
382 + (
383 + "<think>\nprefill",
384 + "\nreason\n</think>\n\nanswer",
385 + "\nreason",
386 + "answer",
387 + ),
388 + ("</think>", "\n\n\nanswer", "", "\nanswer"),
389 + ("</think>\n", "\n\nanswer", "", "\nanswer"),
390 + ("</think>\n\n", "\nanswer", "", "\nanswer"),
391 + ("</think>\n\nprefill", "\nanswer", "", "\nanswer"),
392 + ] {
393 + let mut parser = framed_parser();
394 + parser.initialize(&tokenizer.encode(prompt, false).unwrap()).unwrap();
395 + assert_eq!(
396 + collect_framed(&mut parser, &[wire]),
397 + (reasoning.to_string(), content.to_string()),
398 + "prompt {prompt:?}"
399 + );
400 + }
401 +}
402 +
403 +#[test]
404 +fn delimited_framing_holds_only_boundary_candidates() {
405 + let mut parser = framed_parser();
406 + let first = parser.push(DecodedText::unattributed("<think>\nreason\n"));
407 + assert_eq!(reasoning_str(&first), Some("reason"));
408 + let more = parser.push(DecodedText::unattributed("more\n"));
409 + assert_eq!(reasoning_str(&more), Some("\nmore"));
410 + assert!(parser.push(DecodedText::unattributed("</think>\n")).is_empty());
411 + let answer = parser.push(DecodedText::unattributed("\n answer"));
412 + assert_eq!(content_str(&answer), Some(" answer"));
413 +}
414 +
415 +#[test]
416 +fn delimited_start_prefix_is_consumed_only_before_the_marker() {
417 + for wire in [
418 + "\n<think>reason</think>answer",
419 + "\n\n<think>reason</think>answer",
420 + ] {
421 + for split in 0..=wire.len() {
422 + let mut parser = DelimitedReasoningParserBuilder::new(
423 + Arc::new(fake_tokenizer()),
424 + "<think>",
425 + "</think>",
426 + )
427 + .with_before_start("\n")
428 + .build()
429 + .unwrap();
430 + let expected_content = if wire.starts_with("\n\n") {
431 + "\nanswer"
432 + } else {
433 + "answer"
434 + };
435 + assert_eq!(
436 + collect_framed(&mut parser, &[&wire[..split], &wire[split..]]),
437 + ("reason".to_string(), expected_content.to_string())
438 + );
439 + }
440 + }
441 +}
442 +
443 +#[test]
444 +fn delimited_framing_preserves_body_token_attributions() {
445 + let mut parser = framed_parser();
446 + let mut collected = CollectedAttributions::default();
447 + for chunk in attributed_chunks(&[
448 + Some("<think>"),
449 + Some("\n"),
450 + Some("reason"),
451 + None,
452 + Some("\n"),
453 + Some("</think>"),
454 + Some("\n"),
455 + Some("\n"),
456 + Some(" answer"),
457 + ]) {
458 + collected.record(parser.push(chunk));
459 + }
460 + collected.record(parser.finish());
461 + assert_eq!(collected.reasoning_text, "reason");
462 + assert_eq!(collected.content_text, " answer");
463 + assert_eq!(collected.reasoning_ids, [3, 4]);
464 + assert_eq!(collected.content_ids, [9]);
465 +}
rust/src/parser/src/tool/deepseek_dsml/deepseek_v32.rs
+9
@@ -71,6 +71,7 @@ mod tests {
71
72 use super::DeepSeekV32ToolParser;
73 use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
74 + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
75 use crate::tool::{ToolParser, ToolParserTestExt as _};
76
77 fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
@@ -445,4 +446,12 @@ mod tests {
446 assert_eq!(streamed.normal_text(), complete.normal_text());
447 assert_eq!(streamed.calls(), complete.calls());
448 }
449 +
450 + #[test]
451 + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
452 + assert_tool_framing_preserves_body_whitespace::<DeepSeekV32ToolParser>(
453 + "\n\n",
454 + "<｜DSML｜function_calls><｜DSML｜invoke name=\"get_weather\"></｜DSML｜invoke></｜DSML｜function_calls>",
455 + );
456 + }
457 }
rust/src/parser/src/tool/deepseek_dsml/deepseek_v4.rs
+9
@@ -73,6 +73,7 @@ mod tests {
73
74 use super::DeepSeekV4ToolParser;
75 use crate::tool::test_utils::{collect_stream, test_tools};
76 + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
77 use crate::tool::{ToolParser, ToolParserTestExt as _};
78
79 fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
@@ -145,4 +146,12 @@ mod tests {
146 json!({ "location": "Beijing" })
147 );
148 }
149 +
150 + #[test]
151 + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
152 + assert_tool_framing_preserves_body_whitespace::<DeepSeekV4ToolParser>(
153 + "\n\n",
154 + "<｜DSML｜tool_calls><｜DSML｜invoke name=\"get_weather\"></｜DSML｜invoke></｜DSML｜tool_calls>",
155 + );
156 + }
157 }
rust/src/parser/src/tool/deepseek_dsml/mod.rs
+15 -5
@@ -8,7 +8,7 @@ use winnow::stream::Partial;
8 use winnow::token::{literal, rest, take_until};
9
10 use super::parameters::ToolSchemas;
11 -use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
11 +use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker};
12 use super::{Result, ToolCallDelta, ToolParserOutput};
13 use crate::tool::Tool;
14
@@ -28,16 +28,19 @@ type DsmlInput<'i> = Partial<&'i str>;
28 #[derive(Debug, Clone, Copy)]
29 struct DsmlTokens {
30 tool_calls_start: &'static str,
31 + framed_tool_calls_start: &'static str,
32 tool_calls_end: &'static str,
33 }
34
35 impl DsmlTokens {
36 const V32: Self = Self {
37 tool_calls_start: "<｜DSML｜function_calls>",
38 + framed_tool_calls_start: "\n\n<｜DSML｜function_calls>",
39 tool_calls_end: "</｜DSML｜function_calls>",
40 };
41 const V4: Self = Self {
42 tool_calls_start: "<｜DSML｜tool_calls>",
43 + framed_tool_calls_start: "\n\n<｜DSML｜tool_calls>",
44 tool_calls_end: "</｜DSML｜tool_calls>",
45 };
46 }
@@ -210,9 +213,12 @@ fn parse_tool_block_event(
213
214 /// Parse a DSML function-calls start marker.
215 fn tool_calls_start_event(input: &mut DsmlInput<'_>, tokens: DsmlTokens) -> ModalResult<DsmlEvent> {
213 - literal(tokens.tool_calls_start)
214 - .value(DsmlEvent::ToolCallsStart)
215 - .parse_next(input)
216 + alt((
217 + literal(tokens.framed_tool_calls_start),
218 + literal(tokens.tool_calls_start),
219 + ))
220 + .value(DsmlEvent::ToolCallsStart)
221 + .parse_next(input)
222 }
223
224 /// Parse a DSML function-calls end marker.
@@ -227,7 +233,11 @@ fn ignored_rest_event(input: &mut DsmlInput<'_>) -> ModalResult<DsmlEvent> {
233
234 /// Parse a safe text run before the next DSML marker.
235 fn safe_text_event(input: &mut DsmlInput<'_>, tokens: DsmlTokens) -> ModalResult<DsmlEvent> {
230 - safe_text_len(input, tokens.tool_calls_start).map(|len| DsmlEvent::Text { len })
236 + safe_text_len_mul(
237 + input,
238 + &[tokens.framed_tool_calls_start, tokens.tool_calls_start],
239 + )
240 + .map(|len| DsmlEvent::Text { len })
241 }
242
243 /// Parse a DSML invoke block.
rust/src/parser/src/tool/glm_xml/glm45_moe.rs
+14
@@ -45,3 +45,17 @@ impl ToolParser for Glm45MoeToolParser {
45 self.0.reset()
46 }
47 }
48 +
49 +#[cfg(test)]
50 +mod tests {
51 + use super::Glm45MoeToolParser;
52 + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
53 +
54 + #[test]
55 + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
56 + assert_tool_framing_preserves_body_whitespace::<Glm45MoeToolParser>(
57 + "\n",
58 + "<tool_call>get_weather\n</tool_call>",
59 + );
60 + }
61 +}
rust/src/parser/src/tool/glm_xml/mod.rs
+18 -10
@@ -8,7 +8,7 @@ use winnow::stream::Partial;
8 use winnow::token::{literal, rest, take_until, take_while};
9
10 use super::parameters::ToolSchemas;
11 -use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
11 +use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker};
12 use super::{Result, ToolCallDelta, ToolParserOutput};
13 use crate::tool::Tool;
14
@@ -19,6 +19,7 @@ pub use glm45_moe::Glm45MoeToolParser;
19 pub use glm47_moe::Glm47MoeToolParser;
20
21 const TOOL_CALL_START: &str = "<tool_call>";
22 +const FRAMED_TOOL_CALL_START: &str = "\n<tool_call>";
23 const TOOL_CALL_END: &str = "</tool_call>";
24 const ARG_KEY_START: &str = "<arg_key>";
25 const ARG_KEY_END: &str = "</arg_key>";
@@ -149,7 +150,7 @@ fn parse_next_glm_event(
150 separator: Separator,
151 ) -> ModalResult<GlmEvent> {
152 match mode {
152 - GlmMode::Text => parse_text_event(input),
153 + GlmMode::Text => parse_text_event(input, separator),
154 GlmMode::ToolCall { tool_call_end_scan } => {
155 tool_call_event(input, separator, tool_call_end_scan)
156 }
@@ -158,8 +159,20 @@ fn parse_next_glm_event(
159 }
160
161 /// Parse a text-mode GLM event.
161 -fn parse_text_event(input: &mut GlmInput<'_>) -> ModalResult<GlmEvent> {
162 - alt((tool_call_start_event, safe_text_event)).parse_next(input)
162 +fn parse_text_event(input: &mut GlmInput<'_>, separator: Separator) -> ModalResult<GlmEvent> {
163 + let framed_start = match separator {
164 + Separator::Newline => FRAMED_TOOL_CALL_START,
165 + Separator::Flexible => TOOL_CALL_START,
166 + };
167 + alt((
168 + literal(framed_start).value(GlmEvent::ToolCallStart),
169 + tool_call_start_event,
170 + |input: &mut GlmInput<'_>| {
171 + safe_text_len_mul(input, &[framed_start, TOOL_CALL_START])
172 + .map(|len| GlmEvent::Text { len })
173 + },
174 + ))
175 + .parse_next(input)
176 }
177
178 /// Parse a GLM tool-call start marker.
@@ -167,11 +180,6 @@ fn tool_call_start_event(input: &mut GlmInput<'_>) -> ModalResult<GlmEvent> {
180 literal(TOOL_CALL_START).value(GlmEvent::ToolCallStart).parse_next(input)
181 }
182
170 -/// Parse a safe text run before the next GLM marker.
171 -fn safe_text_event(input: &mut GlmInput<'_>) -> ModalResult<GlmEvent> {
172 - safe_text_len(input, TOOL_CALL_START).map(|len| GlmEvent::Text { len })
173 -}
174 -
183 /// Parse text after a completed GLM tool call.
184 fn after_tool_call_event(input: &mut GlmInput<'_>) -> ModalResult<GlmEvent> {
185 ws0.void().parse_next(input)?;
@@ -303,7 +311,7 @@ mod tests {
311
312 let output = parser.parse_complete(&output).unwrap();
313
306 - assert_eq!(output.normal_text(), "Let me search for that.\n");
314 + assert_eq!(output.normal_text(), "Let me search for that.");
315 assert_eq!(output.calls().len(), 1);
316 assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
317 assert_eq!(
rust/src/parser/src/tool/json/granite4.rs
+1
@@ -222,6 +222,7 @@ fn header_event(input: &mut JsonToolInput<'_>) -> ModalResult<Granite4Event> {
222 const CONFIG: JsonToolCallConfig = JsonToolCallConfig {
223 parser_name: "Granite4",
224 start_marker: "",
225 + framed_start_marker: None,
226 end_marker: "",
227 marker_whitespace: JsonToolCallWhitespace::Optional,
228 delimiter: None,
rust/src/parser/src/tool/json/hermes.rs
+1
@@ -7,6 +7,7 @@ use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutp
7 const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
8 parser_name: "Hermes",
9 start_marker: "<tool_call>",
10 + framed_start_marker: None,
11 end_marker: "</tool_call>",
12 marker_whitespace: JsonToolCallWhitespace::Optional,
13 delimiter: None,
rust/src/parser/src/tool/json/internlm2.rs
+1
@@ -7,6 +7,7 @@ use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
7 const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
8 parser_name: "InternLM2",
9 start_marker: "<|action_start|><|plugin|>",
10 + framed_start_marker: None,
11 end_marker: "<|action_end|>",
12 marker_whitespace: JsonToolCallWhitespace::Optional,
13 delimiter: None,
rust/src/parser/src/tool/json/llama.rs
+1
@@ -208,6 +208,7 @@ fn llama_tool_call_header_event(input: &mut JsonToolInput<'_>) -> ModalResult<Ll
208 const CONFIG: JsonToolCallConfig = JsonToolCallConfig {
209 parser_name: "Llama JSON",
210 start_marker: "",
211 + framed_start_marker: None,
212 end_marker: "",
213 marker_whitespace: JsonToolCallWhitespace::Optional,
214 delimiter: Some(";"),
rust/src/parser/src/tool/json/mistral.rs
+1
@@ -7,6 +7,7 @@ use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
7 const MISTRAL_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
8 parser_name: "Mistral",
9 start_marker: "[TOOL_CALLS] [",
10 + framed_start_marker: None,
11 end_marker: "]",
12 marker_whitespace: JsonToolCallWhitespace::Optional,
13 delimiter: Some(","),
rust/src/parser/src/tool/json/mod.rs
+17 -3
@@ -27,7 +27,8 @@ use winnow::stream::{Partial, Stream};
27 use winnow::token::literal;
28
29 use super::utils::{
30 - JsonObjectScanState, json_str, parse_buffered_event, safe_text_len, take_json_object,
30 + JsonObjectScanState, json_str, parse_buffered_event, safe_text_len, safe_text_len_mul,
31 + take_json_object,
32 };
33 use super::{Result, ToolCallDelta, ToolParserOutput};
34
@@ -37,7 +38,12 @@ pub(crate) type JsonToolInput<'i> = Partial<&'i str>;
38 pub(crate) struct JsonToolCallConfig {
39 pub parser_name: &'static str,
40 pub start_marker: &'static str,
41 + /// Start marker with its fixed preceding framing, matched alongside the
42 + /// bare start marker.
43 + pub framed_start_marker: Option<&'static str>,
44 pub end_marker: &'static str,
45 + /// Whitespace inside the tool-call markers: after the start marker and
46 + /// before the end marker.
47 pub marker_whitespace: JsonToolCallWhitespace,
48 pub delimiter: Option<&'static str>,
49 pub name_key: &'static str,
@@ -211,7 +217,10 @@ fn tool_call_start_event(
217 config: JsonToolCallConfig,
218 ) -> ModalResult<JsonToolCallEvent> {
219 seq!(
214 - _: literal(config.start_marker),
220 + _: |input: &mut JsonToolInput<'_>| match config.framed_start_marker {
221 + Some(marker) => alt((literal(marker), literal(config.start_marker))).void().parse_next(input),
222 + None => literal(config.start_marker).void().parse_next(input),
223 + },
224 _: |input: &mut JsonToolInput<'_>| marker_whitespace(input, config),
225 )
226 .value(JsonToolCallEvent::ToolCallStart)
@@ -369,7 +378,11 @@ fn safe_text_event(
378 input: &mut JsonToolInput<'_>,
379 config: JsonToolCallConfig,
380 ) -> ModalResult<JsonToolCallEvent> {
372 - safe_text_len(input, config.start_marker).map(|len| JsonToolCallEvent::Text { len })
381 + match config.framed_start_marker {
382 + Some(marker) => safe_text_len_mul(input, &[marker, config.start_marker]),
383 + None => safe_text_len(input, config.start_marker),
384 + }
385 + .map(|len| JsonToolCallEvent::Text { len })
386 }
387
388 #[cfg(test)]
@@ -382,6 +395,7 @@ mod tests {
395 const DELIMITED_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
396 parser_name: "Delimited JSON",
397 start_marker: "<tool_calls>",
398 + framed_start_marker: None,
399 end_marker: "</tool_calls>",
400 marker_whitespace: JsonToolCallWhitespace::Optional,
401 delimiter: Some("<"),
rust/src/parser/src/tool/json/phi4mini.rs
+1
@@ -7,6 +7,7 @@ use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
7 const PHI4MINI_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
8 parser_name: "Phi4Mini",
9 start_marker: "functools[",
10 + framed_start_marker: None,
11 end_marker: "]",
12 marker_whitespace: JsonToolCallWhitespace::Optional,
13 delimiter: Some(","),
rust/src/parser/src/tool/json/qwen.rs
+11 -1
@@ -7,6 +7,7 @@ use crate::tool::{Result, StructuralTagBuilder, Tool, ToolParser, ToolParserOutp
7 const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
8 parser_name: "Qwen XML",
9 start_marker: "<tool_call>",
10 + framed_start_marker: Some("\n<tool_call>"),
11 end_marker: "</tool_call>",
12 marker_whitespace: JsonToolCallWhitespace::Exact("\n"),
13 delimiter: None,
@@ -74,6 +75,7 @@ mod tests {
75
76 use super::Qwen3XmlToolParser;
77 use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
78 + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
79 use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
80
81 fn build_tool_call(function_name: &str, arguments: &str) -> String {
@@ -102,7 +104,7 @@ mod tests {
104 ))
105 .unwrap();
106
105 - assert_eq!(output.normal_text(), "Let me check.\n");
107 + assert_eq!(output.normal_text(), "Let me check.");
108 assert_eq!(output.calls().len(), 1);
109 assert_eq!(output.calls()[0].tool_index, 0);
110 assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
@@ -287,4 +289,12 @@ mod tests {
289 expected `name`"#]]
290 .assert_eq(&error.to_report_string());
291 }
292 +
293 + #[test]
294 + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
295 + assert_tool_framing_preserves_body_whitespace::<Qwen3XmlToolParser>(
296 + "\n",
297 + "<tool_call>\n{\"name\":\"get_weather\",\"arguments\":{}}\n</tool_call>",
298 + );
299 + }
300 }
rust/src/parser/src/tool/minimax_m2.rs
+17 -4
@@ -8,11 +8,12 @@ use winnow::stream::Partial;
8 use winnow::token::{literal, rest, take_until};
9
10 use super::parameters::ToolSchemas;
11 -use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
11 +use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker};
12 use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
13 use crate::tool::{StructuralTagBuilder, Tool};
14
15 const TOOL_CALL_START: &str = "<minimax:tool_call>";
16 +const FRAMED_TOOL_CALL_START: &str = "\n<minimax:tool_call>";
17 const TOOL_CALL_END: &str = "</minimax:tool_call>";
18 const INVOKE_START: &str = "<invoke";
19 const INVOKE_END: &str = "</invoke>";
@@ -173,12 +174,15 @@ fn parse_text_event(input: &mut MinimaxM2Input<'_>) -> ModalResult<MinimaxM2Even
174
175 /// Parse a MiniMax M2 tool-block start marker.
176 fn tool_block_start_event(input: &mut MinimaxM2Input<'_>) -> ModalResult<MinimaxM2Event> {
176 - literal(TOOL_CALL_START).value(MinimaxM2Event::ToolBlockStart).parse_next(input)
177 + alt((literal(FRAMED_TOOL_CALL_START), literal(TOOL_CALL_START)))
178 + .value(MinimaxM2Event::ToolBlockStart)
179 + .parse_next(input)
180 }
181
182 /// Parse a safe text run before the next MiniMax M2 marker.
183 fn safe_text_event(input: &mut MinimaxM2Input<'_>) -> ModalResult<MinimaxM2Event> {
181 - safe_text_len(input, TOOL_CALL_START).map(|len| MinimaxM2Event::Text { len })
184 + safe_text_len_mul(input, &[FRAMED_TOOL_CALL_START, TOOL_CALL_START])
185 + .map(|len| MinimaxM2Event::Text { len })
186 }
187
188 /// Parse one event inside a MiniMax M2 tool block.
@@ -277,6 +281,7 @@ mod tests {
281 use super::{MinimaxM2ToolParser, TOOL_CALL_END, TOOL_CALL_START, ToolParser};
282 use crate::tool::ToolParserTestExt as _;
283 use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
284 + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
285
286 fn build_tool_block(invokes: &[(&str, Vec<(&str, &str)>)]) -> String {
287 let invokes = invokes
@@ -543,7 +548,7 @@ mod tests {
548 let mut parser = MinimaxM2ToolParser::new(&test_tools());
549 let result = collect_stream(&mut parser, &chunks);
550
546 - assert_eq!(result.normal_text(), "I will call the tools.\n");
551 + assert_eq!(result.normal_text(), "I will call the tools.");
552 assert_eq!(result.calls().len(), 2);
553 assert_eq!(result.calls()[0].tool_index, 0);
554 assert_eq!(result.calls()[0].name.as_deref(), Some("get_weather"));
@@ -599,4 +604,12 @@ mod tests {
604 expect![[r#"tool parser parsing failed: near "<bad></minimax:tool_call>": "#]]
605 .assert_eq(&error.to_report_string());
606 }
607 +
608 + #[test]
609 + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
610 + assert_tool_framing_preserves_body_whitespace::<MinimaxM2ToolParser>(
611 + "\n",
612 + "<minimax:tool_call><invoke name=\"get_weather\"></invoke></minimax:tool_call>",
613 + );
614 + }
615 }
rust/src/parser/src/tool/qwen_coder.rs
+29 -21
@@ -8,7 +8,7 @@ use winnow::stream::Partial;
8 use winnow::token::{literal, take_until};
9
10 use super::parameters::ToolSchemas;
11 -use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
11 +use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker};
12 use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
13 use crate::tool::{StructuralTagBuilder, Tool};
14
@@ -31,6 +31,8 @@ pub(crate) struct QwenCoderConfig {
31 pub(crate) parser_name: &'static str,
32 /// Marker that opens a tool-call block.
33 pub(crate) tool_call_start: &'static str,
34 + pub(crate) first_tool_call_start: &'static str,
35 + pub(crate) next_tool_call_start: &'static str,
36 /// Marker that closes a tool-call block.
37 pub(crate) tool_call_end: &'static str,
38 }
@@ -38,6 +40,8 @@ pub(crate) struct QwenCoderConfig {
40 const QWEN_CODER_CONFIG: QwenCoderConfig = QwenCoderConfig {
41 parser_name: "Qwen Coder",
42 tool_call_start: TOOL_CALL_START,
43 + first_tool_call_start: "\n\n<tool_call>",
44 + next_tool_call_start: "\n<tool_call>",
45 tool_call_end: TOOL_CALL_END,
46 };
47
@@ -155,7 +159,7 @@ impl ToolParser for Qwen3CoderToolParser {
159 let config = self.config;
160
161 while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| {
158 - parse_next_qwen_coder_event(input, &mut self.mode, config)
162 + parse_next_qwen_coder_event(input, &mut self.mode, config, self.emitted_tool_count > 0)
163 })? {
164 self.apply_event(event, output)?;
165 self.buffer.drain(..consumed_len);
@@ -191,9 +195,10 @@ fn parse_next_qwen_coder_event(
195 input: &mut QwenCoderInput<'_>,
196 mode: &mut QwenCoderMode,
197 config: QwenCoderConfig,
198 + after_tool_call: bool,
199 ) -> ModalResult<QwenCoderEvent> {
200 match mode {
196 - QwenCoderMode::Text => parse_text_event(input, config),
201 + QwenCoderMode::Text => parse_text_event(input, config, after_tool_call),
202 QwenCoderMode::ToolCall { end_marker_scan } => {
203 tool_call_event(input, end_marker_scan, config.tool_call_end)
204 }
@@ -204,30 +209,24 @@ fn parse_next_qwen_coder_event(
209 fn parse_text_event(
210 input: &mut QwenCoderInput<'_>,
211 config: QwenCoderConfig,
212 + after_tool_call: bool,
213 ) -> ModalResult<QwenCoderEvent> {
214 + let framed_start = if after_tool_call {
215 + config.next_tool_call_start
216 + } else {
217 + config.first_tool_call_start
218 + };
219 alt((
209 - |input: &mut QwenCoderInput<'_>| tool_call_start_event(input, config.tool_call_start),
210 - |input: &mut QwenCoderInput<'_>| safe_text_event(input, config.tool_call_start),
220 + alt((literal(framed_start), literal(config.tool_call_start)))
221 + .value(QwenCoderEvent::ToolCallStart),
222 + |input: &mut QwenCoderInput<'_>| {
223 + safe_text_len_mul(input, &[framed_start, config.tool_call_start])
224 + .map(|len| QwenCoderEvent::Text { len })
225 + },
226 ))
227 .parse_next(input)
228 }
229
215 -/// Parse a Qwen Coder tool-call start marker.
216 -fn tool_call_start_event(
217 - input: &mut QwenCoderInput<'_>,
218 - tool_call_start: &'static str,
219 -) -> ModalResult<QwenCoderEvent> {
220 - literal(tool_call_start).value(QwenCoderEvent::ToolCallStart).parse_next(input)
221 -}
222 -
223 -/// Parse a safe text run before the next Qwen Coder marker.
224 -fn safe_text_event(
225 - input: &mut QwenCoderInput<'_>,
226 - tool_call_start: &'static str,
227 -) -> ModalResult<QwenCoderEvent> {
228 - safe_text_len(input, tool_call_start).map(|len| QwenCoderEvent::Text { len })
229 -}
230 -
230 /// Parse a complete Qwen Coder tool call.
231 fn tool_call_event(
232 input: &mut QwenCoderInput<'_>,
@@ -296,6 +295,7 @@ mod tests {
295
296 use super::{Qwen3CoderToolParser, ToolParser};
297 use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
298 + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
299 use crate::tool::{ToolParserOutput, ToolParserTestExt as _};
300
301 fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
@@ -766,4 +766,12 @@ mod tests {
766 json!({ "location": "Hangzhou" })
767 );
768 }
769 +
770 + #[test]
771 + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
772 + assert_tool_framing_preserves_body_whitespace::<Qwen3CoderToolParser>(
773 + "\n\n",
774 + "<tool_call>\n<function=get_weather>\n</function>\n</tool_call>",
775 + );
776 + }
777 }
rust/src/parser/src/tool/seed_oss.rs
+11
@@ -7,6 +7,8 @@ use crate::tool::{Result, Tool, ToolParser, ToolParserOutput};
7 const SEED_OSS_CONFIG: QwenCoderConfig = QwenCoderConfig {
8 parser_name: "Seed-OSS",
9 tool_call_start: "<seed:tool_call>",
10 + first_tool_call_start: "\n\n<seed:tool_call>",
11 + next_tool_call_start: "\n<seed:tool_call>",
12 tool_call_end: "</seed:tool_call>",
13 };
14
@@ -79,6 +81,7 @@ mod tests {
81
82 use super::SeedOssToolParser;
83 use crate::tool::test_utils::{collect_stream, split_by_chars, test_tools};
84 + use crate::tool::tests::assert_tool_framing_preserves_body_whitespace;
85 use crate::tool::{ToolParser, ToolParserOutput, ToolParserTestExt as _};
86
87 fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String {
@@ -250,4 +253,12 @@ mod tests {
253 "tool parser parsing failed: incomplete Seed-OSS tool call"
254 );
255 }
256 +
257 + #[test]
258 + fn tool_framing_preserves_body_whitespace_across_chunk_boundaries() {
259 + assert_tool_framing_preserves_body_whitespace::<SeedOssToolParser>(
260 + "\n\n",
261 + "<seed:tool_call>\n<function=get_weather>\n</function>\n</seed:tool_call>",
262 + );
263 + }
264 }
rust/src/parser/src/tool/tests.rs
+23
@@ -170,3 +170,26 @@ fn default_parse_complete_delegates_through_parse_chunk_and_finish() {
170 ]
171 );
172 }
173 +
174 +pub(super) fn assert_tool_framing_preserves_body_whitespace<P: ToolParser + 'static>(
175 + framing: &str,
176 + call: &str,
177 +) {
178 + use super::test_utils::{collect_stream, test_tools};
179 +
180 + let body = " before\n";
181 + let wire = format!("{body}{framing}{call}");
182 + for split in wire.char_indices().map(|(i, _)| i).chain(std::iter::once(wire.len())) {
183 + let mut parser = P::create(&test_tools()).unwrap();
184 + let output = collect_stream(parser.as_mut(), &[&wire[..split], &wire[split..]]);
185 + assert_eq!(output.normal_text(), body, "split {split}");
186 + assert_eq!(output.calls().len(), 1, "split {split}");
187 + assert_eq!(output.calls()[0].name.as_deref(), Some("get_weather"));
188 + }
189 + let plain = " no tool\n\n<to";
190 + let mut parser = P::create(&test_tools()).unwrap();
191 + assert_eq!(
192 + collect_stream(parser.as_mut(), &[plain]).normal_text(),
193 + plain
194 + );
195 +}
rust/src/parser/src/unified/gemma4.rs
+18 -2
@@ -20,6 +20,7 @@ use crate::utils::{incomplete, parse_buffered_event, partial_prefix_len, safe_te
20 const REASONING_START: &str = "<|channel>thought\n";
21 const CHANNEL_START: &str = "<|channel>";
22 const CHANNEL_END: &str = "<channel|>";
23 +const FRAMED_CHANNEL_END: &str = "\n<channel|>";
24 const TOOL_CALL_START: &str = "<|tool_call>";
25 const TOOL_CALL_END: &str = "<tool_call|>";
26 const STRING_DELIM: &str = "<|\"|>";
@@ -259,7 +260,9 @@ fn reasoning_start_event(input: &mut Gemma4Input<'_>) -> ModalResult<Gemma4Event
260
261 /// Parse a Gemma4 reasoning end marker.
262 fn reasoning_end_event(input: &mut Gemma4Input<'_>) -> ModalResult<Gemma4Event> {
262 - literal(CHANNEL_END).value(Gemma4Event::ReasoningEnd).parse_next(input)
263 + alt((literal(FRAMED_CHANNEL_END), literal(CHANNEL_END)))
264 + .value(Gemma4Event::ReasoningEnd)
265 + .parse_next(input)
266 }
267
268 /// Parse a Gemma4 tool-call start marker.
@@ -308,7 +311,8 @@ fn safe_text_event(input: &mut Gemma4Input<'_>) -> ModalResult<Gemma4Event> {
311
312 /// Parse a safe reasoning run before the next Gemma4 marker.
313 fn safe_reasoning_event(input: &mut Gemma4Input<'_>) -> ModalResult<Gemma4Event> {
311 - safe_text_len_mul(input, &[CHANNEL_END, TOOL_CALL_START]).map(|_| Gemma4Event::Reasoning)
314 + safe_text_len_mul(input, &[FRAMED_CHANNEL_END, CHANNEL_END, TOOL_CALL_START])
315 + .map(|_| Gemma4Event::Reasoning)
316 }
317
318 /// Parse raw Gemma4 arguments through the first end marker outside a Gemma string.
@@ -686,6 +690,18 @@ mod tests {
690 output.coalesce()
691 }
692
693 + #[test]
694 + fn gemma4_reasoning_framing_preserves_extra_newlines_at_every_split() {
695 + let wire = "<|channel>thought\n\n reason\n\n<channel|> answer\n";
696 + for split in 0..=wire.len() {
697 + let output = collect_stream(&[&wire[..split], &wire[split..]]);
698 + assert_eq!(output.reasoning_text(), "\n reason\n", "split {split}");
699 + assert_eq!(output.normal_text(), " answer\n", "split {split}");
700 + }
701 + let output = collect_stream(&["<|channel>thought\nreason\n<channel"]);
702 + assert_eq!(output.reasoning_text(), "reason\n<channel");
703 + }
704 +
705 fn first_call(output: &UnifiedParserOutput) -> ToolCallDelta {
706 (*output.calls().first().expect("expected one tool call")).clone()
707 }
rust/src/parser/src/unified/inkling.rs
+1
@@ -45,6 +45,7 @@ const BLOCK_END_MARKERS: &[&str] = &[END_MESSAGE, CONTENT_MODEL_END_SAMPLING];
45 const INKLING_TOOL_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
46 parser_name: "Inkling",
47 start_marker: CONTENT_INVOKE_TOOL_JSON,
48 + framed_start_marker: None,
49 end_marker: END_MESSAGE,
50 marker_whitespace: JsonToolCallWhitespace::Optional,
51 delimiter: None,