@tduncan / Nvidia-Optimization-Stack-VLLM / commits / df3b3422b4

[Rust Frontend] Add HY3 unified parser and local XGrammar structural-tag builder (#53054)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Aug 21, 2026 at 12:13 UTC df3b3422b495a33fd10d0b2f06c051144d6bb2c2
17 files changed +849 -86
rust/src/chat/src/lib.rs
+1 -1
@@ -415,6 +415,6 @@ mod tests {
415 )
416 .unwrap_err();
417
418 - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
418 + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, hy_v3, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
419 }
420 }
rust/src/chat/src/parser/reasoning/mod.rs
+3
@@ -25,6 +25,7 @@ pub mod names {
25 pub const GEMMA4: &str = "gemma4";
26 pub const INKLING: &str = "inkling";
27 pub const GLM45: &str = "glm45";
28 + pub const HY_V3: &str = "hy_v3";
29 pub const KIMI: &str = "kimi";
30 pub const KIMI_K2: &str = "kimi_k2";
31 pub const KIMI_K3: &str = "kimi_k3";
@@ -67,6 +68,7 @@ impl ReasoningParserFactory {
68 .register_unified_dummy(names::GEMMA4)
69 .register_unified_dummy(names::INKLING)
70 .register_parser::<Glm45ReasoningParser>(names::GLM45)
71 + .register_unified_dummy(names::HY_V3)
72 .register_parser::<KimiReasoningParser>(names::KIMI)
73 .register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
74 .register_unified_dummy(names::KIMI_K3)
@@ -91,6 +93,7 @@ impl ReasoningParserFactory {
93 .register_pattern("glm-4.7", names::GLM45)
94 .register_pattern("glm-4.6", names::GLM45)
95 .register_pattern("glm-4.5", names::GLM45)
96 + .register_pattern("hy3", names::HY_V3)
97 .register_pattern("kimi-k2", names::KIMI_K2)
98 .register_pattern("kimi", names::KIMI)
99 // step3p5 patterns must precede `step3`: substring matching would
rust/src/chat/src/parser/tool/mod.rs
+2 -3
@@ -7,7 +7,7 @@ use std::sync::{Arc, LazyLock};
7
8 pub use vllm_parser::tool::{
9 DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
10 - Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser, HyV3ToolParser,
10 + Glm45MoeToolParser, Glm47MoeToolParser, Granite4ToolParser, HermesToolParser,
11 Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser,
12 MinimaxM3ToolParser, MistralToolParser, Phi4MiniJsonToolParser, Qwen3CoderToolParser,
13 Qwen3XmlToolParser, SeedOssToolParser, ToolParser, ToolParserError,
@@ -76,7 +76,7 @@ impl ToolParserFactory {
76 .register_unified_dummy(names::INKLING)
77 .register_parser::<Granite4ToolParser>(names::GRANITE4)
78 .register_parser::<HermesToolParser>(names::HERMES)
79 - .register_parser::<HyV3ToolParser>(names::HY_V3)
79 + .register_unified_dummy(names::HY_V3)
80 .register_parser::<Internlm2ToolParser>(names::INTERNLM)
81 .register_parser::<KimiK2ToolParser>(names::KIMI_K2)
82 .register_unified_dummy(names::KIMI_K3)
@@ -100,7 +100,6 @@ impl ToolParserFactory {
100 .register_pattern("qwen3", names::QWEN3_XML)
101 .register_pattern("hermes", names::HERMES)
102 .register_pattern("hy3", names::HY_V3)
103 - .register_pattern("hy_v3", names::HY_V3)
103 // Narrow to `internlm2` substring so it matches `internlm2-chat-7b`
104 // and `internlm2_5-7b-chat` but NOT `internlm-chat-7b` (InternLM v1,
105 // routes to Llama), `internlm3-*` (also Llama-architecture per
rust/src/chat/src/parser/unified.rs
+5 -1
@@ -6,7 +6,8 @@
6 use std::sync::LazyLock;
7
8 pub use vllm_parser::unified::{
9 - Gemma4UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, UnifiedParser,
9 + Gemma4UnifiedParser, HyV3UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser,
10 + UnifiedParser,
11 };
12 use vllm_tokenizer::DynTokenizer;
13
@@ -16,6 +17,7 @@ use crate::request::ChatTool;
17 /// Canonical public names for registered unified parsers.
18 pub mod names {
19 pub const GEMMA4: &str = "gemma4";
20 + pub const HY_V3: &str = "hy_v3";
21 pub const INKLING: &str = "inkling";
22 pub const KIMI_K3: &str = "kimi_k3";
23 }
@@ -41,12 +43,14 @@ impl UnifiedParserFactory {
43 let mut factory = Self::default();
44
45 factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
46 + factory.register_parser::<HyV3UnifiedParser>(names::HY_V3);
47 factory.register_parser::<InklingUnifiedParser>(names::INKLING);
48 factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);
49
50 factory
51 .register_pattern("gemma-4", names::GEMMA4)
52 .register_pattern("gemma4", names::GEMMA4)
53 + .register_pattern("hy3", names::HY_V3)
54 .register_pattern("inkling", names::INKLING)
55 .register_pattern("kimi-k3", names::KIMI_K3)
56 .register_pattern("kimi_k3", names::KIMI_K3);
rust/src/chat/tests/roundtrip.rs
+60 -7
@@ -55,6 +55,12 @@ enum ThinkingBehavior {
55 Toggleable { default: bool },
56 /// The chat template always behaves as `value` for this fixture.
57 Always { value: bool },
58 + /// The chat template selects thinking mode through `reasoning_effort`.
59 + ReasoningEffort {
60 + default: bool,
61 + enabled: &'static str,
62 + disabled: &'static str,
63 + },
64 }
65
66 impl ThinkingBehavior {
@@ -62,6 +68,7 @@ impl ThinkingBehavior {
68 match self {
69 Self::Toggleable { default } => default,
70 Self::Always { value } => value,
71 + Self::ReasoningEffort { default, .. } => default,
72 }
73 }
74
@@ -76,6 +83,33 @@ impl ThinkingBehavior {
83 Some(value), // explicitly request the supported thinking behavior
84 None, // use default template behavior
85 ],
86 + Self::ReasoningEffort { .. } => vec![
87 + Some(true), // explicitly enable thinking
88 + Some(false), // explicitly disable thinking
89 + None, // use default template behavior
90 + ],
91 + }
92 + }
93 +
94 + fn apply(self, request: &mut ChatRequest, thinking: Option<bool>) {
95 + let Some(thinking) = thinking else {
96 + return;
97 + };
98 +
99 + match self {
100 + Self::Toggleable { .. } | Self::Always { .. } => {
101 + for key in ["thinking", "enable_thinking"] {
102 + request.chat_options.template_kwargs.insert(key.to_string(), thinking.into());
103 + }
104 + }
105 + Self::ReasoningEffort {
106 + enabled, disabled, ..
107 + } => {
108 + request.chat_options.template_kwargs.insert(
109 + "reasoning_effort".to_string(),
110 + if thinking { enabled } else { disabled }.into(),
111 + );
112 + }
113 }
114 }
115 }
@@ -242,6 +276,23 @@ impl RoundtripCase {
276 }
277 }
278
279 + /// HY3 suffixed reasoning/tool markers discovered from tokenizer added vocab.
280 + fn hy_v3() -> Self {
281 + Self {
282 + model_id: "tencent/Hy3",
283 + assistant_stop_suffix: "<｜hy_eos:opensource｜>",
284 + tool_call_parser: ParserSelection::Auto,
285 + reasoning_parser: ParserSelection::Auto,
286 + thinking_behavior: ThinkingBehavior::ReasoningEffort {
287 + default: false,
288 + enabled: "high",
289 + disabled: "no_think",
290 + },
291 + json_fmt: compact_json_fmt(),
292 + sort_json_keys: false,
293 + }
294 + }
295 +
296 /// SeedOSS with `<seed:think>` / `</seed:think>` reasoning tags.
297 fn seed_oss() -> Self {
298 Self {
@@ -345,6 +396,9 @@ roundtrip_tests! {
396 kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
397 // K3 drops plain-assistant reasoning in history; tool-call turns keep it.
398 kimi_k3 => [tool_call_mix],
399 + // HY3's final plain-assistant history omits EOS; tool-call history keeps it
400 + // and exercises both stages of the tokenizer-derived unified parser.
401 + hy_v3 => [tool_call_mix],
402 gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call
403 inkling => [reasoning_and_content, tool_call_mix],
404 }
@@ -370,6 +424,7 @@ async fn run_roundtrip_reasoning_and_content_inner(
424 vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")],
425 Vec::new(),
426 thinking,
427 + case.thinking_behavior,
428 );
429 let expected_reasoning = "Need compute 2 + 2 directly.";
430 let expected_text = "The answer is 4.";
@@ -417,6 +472,7 @@ async fn run_roundtrip_tool_call_mix(
472 )],
473 test_tools(),
474 Some(true), // always enable thinking in this fixture
475 + case.thinking_behavior,
476 );
477 let expected_reasoning = "Need call the weather and add tools.";
478 let expected_text = "I will call the tools.";
@@ -838,6 +894,7 @@ fn roundtrip_request(
894 messages: Vec<ChatMessage>,
895 tools: Vec<ChatTool>,
896 thinking: Option<bool>,
897 + thinking_behavior: ThinkingBehavior,
898 ) -> ChatRequest {
899 let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, None, true)
900 .expect("tool context should resolve");
@@ -848,13 +905,9 @@ fn roundtrip_request(
905 ..ChatRequest::for_test()
906 };
907
851 - // Explicitly enable or disable thinking so that rendering and parsing the reasoning block is
852 - // exercised or skipped in the roundtrip. If unspecified, use the default template behavior.
853 - if let Some(thinking) = thinking {
854 - for key in ["thinking", "enable_thinking"] {
855 - request.chat_options.template_kwargs.insert(key.to_string(), thinking.into());
856 - }
857 - }
908 + // Explicitly enable or disable thinking using the controls understood by
909 + // this model's template. If unspecified, use the template default.
910 + thinking_behavior.apply(&mut request, thinking);
911
912 request
913 }
rust/src/parser/src/reasoning/delimited.rs
+13 -9
@@ -35,25 +35,29 @@ impl DelimitedReasoningParser {
35 /// start or end delimiter, that prompt boundary always wins.
36 pub(crate) fn new(
37 tokenizer: DynTokenizer,
38 - start_token: &'static str,
39 - end_token: &'static str,
38 + start_token: impl Into<String>,
39 + end_token: impl Into<String>,
40 default_in_reasoning: bool,
41 ) -> Result<Self> {
42 + let start_token = start_token.into();
43 + let end_token = end_token.into();
44 let start_token_id =
43 - tokenizer.token_to_id(start_token).ok_or_else(|| ReasoningError::MissingToken {
44 - token: start_token.to_string(),
45 - })?;
45 + tokenizer
46 + .token_to_id(&start_token)
47 + .ok_or_else(|| ReasoningError::MissingToken {
48 + token: start_token.clone(),
49 + })?;
50 let end_token_id =
47 - tokenizer.token_to_id(end_token).ok_or_else(|| ReasoningError::MissingToken {
48 - token: end_token.to_string(),
51 + tokenizer.token_to_id(&end_token).ok_or_else(|| ReasoningError::MissingToken {
52 + token: end_token.clone(),
53 })?;
54
55 Ok(Self {
56 tokenizer,
57 current_in_reasoning: default_in_reasoning,
58 buffer: String::new(),
55 - start_token: start_token.to_string(),
56 - end_token: end_token.to_string(),
59 + start_token,
60 + end_token,
61 start_token_id,
62 end_token_id,
63 default_in_reasoning,
rust/src/parser/src/reasoning/hy_v3.rs new
+51
@@ -0,0 +1,51 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +use vllm_tokenizer::DynTokenizer;
5 +
6 +use super::{DelimitedReasoningParser, ReasoningDelta, ReasoningError, ReasoningParser, Result};
7 +
8 +/// Internal HY3 reasoning stage used by the unified HY3 parser.
9 +pub(crate) struct HyV3ReasoningParser {
10 + inner: DelimitedReasoningParser,
11 +}
12 +
13 +impl HyV3ReasoningParser {
14 + /// Create a HY3 reasoning parser for the tokenizer-specific marker suffix.
15 + pub(crate) fn new(tokenizer: DynTokenizer, suffix: &str) -> Result<Self> {
16 + Ok(Self {
17 + inner: DelimitedReasoningParser::new(
18 + tokenizer,
19 + format!("<think{suffix}>"),
20 + format!("</think{suffix}>"),
21 + false,
22 + )?,
23 + })
24 + }
25 +}
26 +
27 +impl ReasoningParser for HyV3ReasoningParser {
28 + // Suffix discovery belongs to `HyV3UnifiedParser` so its reasoning and
29 + // tool delimiters always use the same tokenizer-derived value.
30 + fn create(_tokenizer: DynTokenizer) -> Result<Box<dyn ReasoningParser>>
31 + where
32 + Self: Sized + 'static,
33 + {
34 + Err(ReasoningError::DummyUnifiedParser {
35 + name: "hy_v3".to_string(),
36 + })
37 + }
38 +
39 + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
40 + self.inner.initialize(prompt_token_ids);
41 + Ok(())
42 + }
43 +
44 + fn push(&mut self, delta: &str) -> Result<ReasoningDelta> {
45 + Ok(self.inner.push(delta))
46 + }
47 +
48 + fn finish(&mut self) -> Result<ReasoningDelta> {
49 + Ok(self.inner.finish())
50 + }
51 +}
rust/src/parser/src/reasoning/mod.rs
+2
@@ -20,6 +20,7 @@
20 mod cohere_cmd;
21 mod deepseek_r1;
22 mod delimited;
23 +mod hy_v3;
24 mod kimi;
25 mod minimax_m3;
26 mod qwen3;
@@ -32,6 +33,7 @@ use vllm_tokenizer::DynTokenizer;
33 pub use self::cohere_cmd::CohereCmdReasoningParser;
34 pub use self::deepseek_r1::DeepSeekR1ReasoningParser;
35 pub(crate) use self::delimited::{DelimitedReasoningParser, last_reasoning_boundary};
36 +pub(crate) use self::hy_v3::HyV3ReasoningParser;
37 pub use self::kimi::KimiReasoningParser;
38 pub use self::minimax_m3::MiniMaxM3ReasoningParser;
39 pub use self::qwen3::Qwen3ReasoningParser;
rust/src/parser/src/tool/hy_v3.rs
+142 -62
@@ -1,26 +1,65 @@
1 // SPDX-License-Identifier: Apache-2.0
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +mod structural_tag;
5 +
6 +use std::sync::Arc;
7 +
8 use winnow::ascii::multispace0 as ws0;
9 use winnow::combinator::{alt, delimited, eof, repeat, seq, terminated};
10 use winnow::prelude::*;
11 use winnow::stream::Partial;
12 use winnow::token::{literal, rest, take_until};
13
14 +use self::structural_tag::HyV3StructuralTagBuilder;
15 use super::parameters::ToolSchemas;
16 use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker};
12 -use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput};
17 +use super::{Result, ToolCallDelta, ToolParser, ToolParserError, ToolParserOutput};
18 use crate::tool::{StructuralTagBuilder, Tool};
19
15 -const TOOL_CALLS_START: &str = "<tool_calls>";
16 -const TOOL_CALLS_END: &str = "</tool_calls>";
17 -const TOOL_CALL_START: &str = "<tool_call>";
18 -const TOOL_CALL_END: &str = "</tool_call>";
19 -const TOOL_SEP: &str = "<tool_sep>";
20 -const ARG_KEY_START: &str = "<arg_key>";
21 -const ARG_KEY_END: &str = "</arg_key>";
22 -const ARG_VALUE_START: &str = "<arg_value>";
23 -const ARG_VALUE_END: &str = "</arg_value>";
20 +#[derive(Debug, Clone, PartialEq, Eq)]
21 +pub(crate) struct HyV3ToolMarkers {
22 + tool_calls_start: String,
23 + tool_calls_end: String,
24 + tool_call_start: String,
25 + tool_call_end: String,
26 + tool_sep: String,
27 + arg_key_start: String,
28 + arg_key_end: String,
29 + arg_value_start: String,
30 + arg_value_end: String,
31 +}
32 +
33 +impl HyV3ToolMarkers {
34 + pub(crate) fn new(suffix: &str) -> Self {
35 + Self {
36 + tool_calls_start: format!("<tool_calls{suffix}>"),
37 + tool_calls_end: format!("</tool_calls{suffix}>"),
38 + tool_call_start: format!("<tool_call{suffix}>"),
39 + tool_call_end: format!("</tool_call{suffix}>"),
40 + tool_sep: format!("<tool_sep{suffix}>"),
41 + arg_key_start: format!("<arg_key{suffix}>"),
42 + arg_key_end: format!("</arg_key{suffix}>"),
43 + arg_value_start: format!("<arg_value{suffix}>"),
44 + arg_value_end: format!("</arg_value{suffix}>"),
45 + }
46 + }
47 +
48 + pub(crate) fn iter(&self) -> impl Iterator<Item = &str> {
49 + [
50 + self.tool_calls_start.as_str(),
51 + self.tool_calls_end.as_str(),
52 + self.tool_call_start.as_str(),
53 + self.tool_call_end.as_str(),
54 + self.tool_sep.as_str(),
55 + self.arg_key_start.as_str(),
56 + self.arg_key_end.as_str(),
57 + self.arg_value_start.as_str(),
58 + self.arg_value_end.as_str(),
59 + ]
60 + .into_iter()
61 + }
62 +}
63
64 type HyV3Input<'i> = Partial<&'i str>;
65
@@ -60,21 +99,26 @@ enum HyV3Event {
99 /// Arguments are emitted only after a full `<tool_call>` block is parsed.
100 /// HY3 marker tokens are added-vocabulary tokens rather than tokenizer special
101 /// tokens, so the default `preserve_special_tokens() == false` is sufficient.
63 -pub struct HyV3ToolParser {
102 +pub(crate) struct HyV3ToolParser {
103 buffer: String,
104 mode: HyV3Mode,
105 emitted_tool_count: usize,
106 tool_parameters: ToolSchemas,
107 + markers: Arc<HyV3ToolMarkers>,
108 + structural_tag_builder: HyV3StructuralTagBuilder,
109 }
110
111 impl HyV3ToolParser {
112 /// Create a HY3 tool parser.
72 - fn new(tools: &[Tool]) -> Self {
113 + pub(crate) fn new(tools: &[Tool], suffix: &str) -> Self {
114 + let markers = Arc::new(HyV3ToolMarkers::new(suffix));
115 Self {
116 buffer: String::new(),
117 mode: HyV3Mode::Text,
118 emitted_tool_count: 0,
119 tool_parameters: ToolSchemas::from_tools(tools),
120 + markers: Arc::clone(&markers),
121 + structural_tag_builder: HyV3StructuralTagBuilder::new(markers),
122 }
123 }
124
@@ -109,22 +153,26 @@ impl HyV3ToolParser {
153 }
154
155 impl ToolParser for HyV3ToolParser {
112 - fn create(tools: &[Tool]) -> Result<Box<dyn ToolParser>>
156 + // Suffix discovery belongs to `HyV3UnifiedParser` so its reasoning and
157 + // tool delimiters always use the same tokenizer-derived value.
158 + fn create(_tools: &[Tool]) -> Result<Box<dyn ToolParser>>
159 where
160 Self: Sized + 'static,
161 {
116 - Ok(Box::new(Self::new(tools)))
162 + Err(ToolParserError::DummyUnifiedParser {
163 + name: "hy_v3".to_string(),
164 + })
165 }
166
167 fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
120 - Some(xgrammar_structural_tag::Model::HyV3.builder())
168 + Some(&self.structural_tag_builder)
169 }
170
171 fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
172 self.buffer.push_str(chunk);
173
174 while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| {
127 - parse_next_hy_v3_event(input, &mut self.mode)
175 + parse_next_hy_v3_event(input, &mut self.mode, &self.markers)
176 })? {
177 self.apply_event(event, output)?;
178 self.buffer.drain(..consumed_len);
@@ -155,62 +203,83 @@ impl ToolParser for HyV3ToolParser {
203 fn parse_next_hy_v3_event(
204 input: &mut HyV3Input<'_>,
205 mode: &mut HyV3Mode,
206 + markers: &HyV3ToolMarkers,
207 ) -> ModalResult<HyV3Event> {
208 match mode {
160 - HyV3Mode::Text => parse_text_event(input),
209 + HyV3Mode::Text => parse_text_event(input, markers),
210 HyV3Mode::ToolBlock { tool_call_end_scan } => {
162 - parse_tool_block_event(input, tool_call_end_scan)
211 + parse_tool_block_event(input, tool_call_end_scan, markers)
212 }
213 HyV3Mode::Done => ignored_rest_event(input),
214 }
215 }
216
217 /// Parse a text-mode HY3 event.
169 -fn parse_text_event(input: &mut HyV3Input<'_>) -> ModalResult<HyV3Event> {
170 - alt((tool_block_start_event, safe_text_event)).parse_next(input)
218 +fn parse_text_event(
219 + input: &mut HyV3Input<'_>,
220 + markers: &HyV3ToolMarkers,
221 +) -> ModalResult<HyV3Event> {
222 + alt((
223 + |input: &mut HyV3Input<'_>| tool_block_start_event(input, markers),
224 + |input: &mut HyV3Input<'_>| safe_text_event(input, markers),
225 + ))
226 + .parse_next(input)
227 }
228
229 /// Parse a HY3 tool-block start marker.
174 -fn tool_block_start_event(input: &mut HyV3Input<'_>) -> ModalResult<HyV3Event> {
175 - literal(TOOL_CALLS_START).value(HyV3Event::ToolBlockStart).parse_next(input)
230 +fn tool_block_start_event(
231 + input: &mut HyV3Input<'_>,
232 + markers: &HyV3ToolMarkers,
233 +) -> ModalResult<HyV3Event> {
234 + literal(markers.tool_calls_start.as_str())
235 + .value(HyV3Event::ToolBlockStart)
236 + .parse_next(input)
237 }
238
239 /// Parse a safe text run before the next HY3 marker.
179 -fn safe_text_event(input: &mut HyV3Input<'_>) -> ModalResult<HyV3Event> {
180 - safe_text_len(input, TOOL_CALLS_START).map(|len| HyV3Event::Text { len })
240 +fn safe_text_event(input: &mut HyV3Input<'_>, markers: &HyV3ToolMarkers) -> ModalResult<HyV3Event> {
241 + safe_text_len(input, &markers.tool_calls_start).map(|len| HyV3Event::Text { len })
242 }
243
244 /// Parse one event inside a HY3 tool block.
245 fn parse_tool_block_event(
246 input: &mut HyV3Input<'_>,
247 tool_call_end_scan: &mut MarkerScanState,
248 + markers: &HyV3ToolMarkers,
249 ) -> ModalResult<HyV3Event> {
188 - alt((tool_block_end_event, |input: &mut HyV3Input<'_>| {
189 - tool_call_event(input, tool_call_end_scan)
190 - }))
250 + alt((
251 + |input: &mut HyV3Input<'_>| tool_block_end_event(input, markers),
252 + |input: &mut HyV3Input<'_>| tool_call_event(input, tool_call_end_scan, markers),
253 + ))
254 .parse_next(input)
255 }
256
257 /// Parse a HY3 tool-block end marker.
195 -fn tool_block_end_event(input: &mut HyV3Input<'_>) -> ModalResult<HyV3Event> {
196 - (ws0, literal(TOOL_CALLS_END)).value(HyV3Event::ToolBlockEnd).parse_next(input)
258 +fn tool_block_end_event(
259 + input: &mut HyV3Input<'_>,
260 + markers: &HyV3ToolMarkers,
261 +) -> ModalResult<HyV3Event> {
262 + (ws0, literal(markers.tool_calls_end.as_str()))
263 + .value(HyV3Event::ToolBlockEnd)
264 + .parse_next(input)
265 }
266
267 /// Parse a complete HY3 tool-call block.
268 fn tool_call_event(
269 input: &mut HyV3Input<'_>,
270 tool_call_end_scan: &mut MarkerScanState,
271 + markers: &HyV3ToolMarkers,
272 ) -> ModalResult<HyV3Event> {
273 let (name, body) = seq!(
274 _: ws0,
206 - _: literal(TOOL_CALL_START),
207 - take_until(0.., TOOL_SEP),
208 - _: literal(TOOL_SEP),
209 - take_until_marker(TOOL_CALL_END, tool_call_end_scan),
210 - _: literal(TOOL_CALL_END),
275 + _: literal(markers.tool_call_start.as_str()),
276 + take_until(0.., markers.tool_sep.as_str()),
277 + _: literal(markers.tool_sep.as_str()),
278 + take_until_marker(markers.tool_call_end.as_str(), tool_call_end_scan),
279 + _: literal(markers.tool_call_end.as_str()),
280 )
281 .parse_next(input)?;
213 - let raw_params = parse_tool_call_params(body)?;
282 + let raw_params = parse_tool_call_params(body, markers)?;
283
284 Ok(HyV3Event::ToolCall {
285 name: name.trim().to_string(),
@@ -219,21 +288,32 @@ fn tool_call_event(
288 }
289
290 /// Parse all parameter blocks inside a complete HY3 tool call.
222 -fn parse_tool_call_params(tool_call_body: &str) -> ModalResult<Vec<(String, String)>> {
291 +fn parse_tool_call_params(
292 + tool_call_body: &str,
293 + markers: &HyV3ToolMarkers,
294 +) -> ModalResult<Vec<(String, String)>> {
295 let mut input = tool_call_body;
224 - delimited(ws0, repeat(0.., terminated(parameter, ws0)), eof).parse_next(&mut input)
296 + delimited(
297 + ws0,
298 + repeat(
299 + 0..,
300 + terminated(|input: &mut &str| parameter(input, markers), ws0),
301 + ),
302 + eof,
303 + )
304 + .parse_next(&mut input)
305 }
306
307 /// Parse a HY3 argument key/value block.
228 -fn parameter(input: &mut &str) -> ModalResult<(String, String)> {
308 +fn parameter(input: &mut &str, markers: &HyV3ToolMarkers) -> ModalResult<(String, String)> {
309 let (name, value) = seq!(
230 - _: literal(ARG_KEY_START),
231 - take_until(0.., ARG_KEY_END),
232 - _: literal(ARG_KEY_END),
310 + _: literal(markers.arg_key_start.as_str()),
311 + take_until(0.., markers.arg_key_end.as_str()),
312 + _: literal(markers.arg_key_end.as_str()),
313 _: ws0,
234 - _: literal(ARG_VALUE_START),
235 - take_until(0.., ARG_VALUE_END),
236 - _: literal(ARG_VALUE_END),
314 + _: literal(markers.arg_value_start.as_str()),
315 + take_until(0.., markers.arg_value_end.as_str()),
316 + _: literal(markers.arg_value_end.as_str()),
317 )
318 .parse_next(input)?;
319
@@ -274,14 +354,14 @@ mod tests {
354
355 #[test]
356 fn hy_v3_does_not_preserve_special_tokens() {
277 - let parser = HyV3ToolParser::new(&test_tools());
357 + let parser = HyV3ToolParser::new(&test_tools(), "");
358
359 assert!(!parser.preserve_special_tokens());
360 }
361
362 #[test]
363 fn hy_v3_parse_complete_without_tool_call_keeps_text() {
284 - let mut parser = HyV3ToolParser::new(&test_tools());
364 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
365 let output = parser.parse_complete("This is a plain response.").unwrap();
366
367 assert_eq!(output.normal_text(), "This is a plain response.");
@@ -290,7 +370,7 @@ mod tests {
370
371 #[test]
372 fn hy_v3_parse_complete_extracts_zero_arg_inline_tool_call() {
293 - let mut parser = HyV3ToolParser::new(&test_tools());
373 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
374 let output = parser
375 .parse_complete(
376 "<tool_calls><tool_call>get_current_date<tool_sep></tool_call></tool_calls>",
@@ -305,7 +385,7 @@ mod tests {
385
386 #[test]
387 fn hy_v3_parse_complete_extracts_zero_arg_newline_tool_call() {
308 - let mut parser = HyV3ToolParser::new(&test_tools());
388 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
389 let output = parser
390 .parse_complete(
391 "<tool_calls>\n<tool_call>get_current_date<tool_sep>\n</tool_call>\n</tool_calls>",
@@ -318,7 +398,7 @@ mod tests {
398
399 #[test]
400 fn hy_v3_parse_complete_extracts_arguments_on_same_line() {
321 - let mut parser = HyV3ToolParser::new(&test_tools());
401 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
402 let output = parser
403 .parse_complete(
404 "<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Beijing</arg_value><arg_key>date</arg_key><arg_value>2026-03-30</arg_value></tool_call></tool_calls>",
@@ -333,7 +413,7 @@ mod tests {
413
414 #[test]
415 fn hy_v3_parse_complete_extracts_arguments_with_newlines() {
336 - let mut parser = HyV3ToolParser::new(&test_tools());
416 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
417 let output = parser
418 .parse_complete(&build_tool_calls(&[build_tool_call(
419 "get_weather",
@@ -349,7 +429,7 @@ mod tests {
429
430 #[test]
431 fn hy_v3_parse_complete_preserves_prefix_and_ignores_trailing_text() {
352 - let mut parser = HyV3ToolParser::new(&test_tools());
432 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
433 let output = parser
434 .parse_complete(&format!(
435 "Checking.{} trailing text",
@@ -363,7 +443,7 @@ mod tests {
443
444 #[test]
445 fn hy_v3_parse_complete_extracts_multiple_tool_calls_in_one_block() {
366 - let mut parser = HyV3ToolParser::new(&test_tools());
446 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
447 let output = parser
448 .parse_complete(&build_tool_calls(&[
449 build_tool_call(
@@ -406,7 +486,7 @@ mod tests {
486
487 #[test]
488 fn hy_v3_parse_complete_converts_schema_types() {
409 - let mut parser = HyV3ToolParser::new(&test_tools());
489 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
490 let output = parser
491 .parse_complete(&build_tool_calls(&[build_tool_call(
492 "convert",
@@ -432,7 +512,7 @@ mod tests {
512
513 #[test]
514 fn hy_v3_streaming_without_tool_call_emits_text_incrementally() {
435 - let mut parser = HyV3ToolParser::new(&test_tools());
515 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
516 let mut output = ToolParserOutput::default();
517
518 output.append(parser.parse_chunk("This is ").unwrap());
@@ -446,7 +526,7 @@ mod tests {
526
527 #[test]
528 fn hy_v3_streaming_extracts_zero_arg_tool_call() {
449 - let mut parser = HyV3ToolParser::new(&test_tools());
529 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
530 let chunks = [
531 "<tool_calls>",
532 "\n<tool_call>",
@@ -465,7 +545,7 @@ mod tests {
545
546 #[test]
547 fn hy_v3_streaming_extracts_arguments() {
468 - let mut parser = HyV3ToolParser::new(&test_tools());
548 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
549 let chunks = [
550 "<tool_calls>",
551 "\n<tool_call>",
@@ -491,7 +571,7 @@ mod tests {
571
572 #[test]
573 fn hy_v3_streaming_preserves_prefix_text() {
494 - let mut parser = HyV3ToolParser::new(&test_tools());
574 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
575 let chunks = [
576 "Checking.",
577 "<tool_calls>",
@@ -521,7 +601,7 @@ mod tests {
601 ),
602 ]);
603 let chunks = split_by_chars(&input, 9);
524 - let mut parser = HyV3ToolParser::new(&test_tools());
604 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
605
606 let output = collect_stream(&mut parser, &chunks);
607
@@ -537,7 +617,7 @@ mod tests {
617 build_tool_calls(&[build_tool_call("get_weather", &[("city", "Beijing")])])
618 );
619 let chunks = split_by_chars(&input, 5);
540 - let mut parser = HyV3ToolParser::new(&test_tools());
620 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
621
622 let output = collect_stream(&mut parser, &chunks);
623
@@ -548,7 +628,7 @@ mod tests {
628
629 #[test]
630 fn hy_v3_streaming_does_not_emit_incomplete_tool_call() {
551 - let mut parser = HyV3ToolParser::new(&test_tools());
631 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
632 let mut output = ToolParserOutput::default();
633
634 parser
@@ -564,7 +644,7 @@ mod tests {
644
645 #[test]
646 fn hy_v3_finish_fails_incomplete_tool_call() {
567 - let mut parser = HyV3ToolParser::new(&test_tools());
647 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
648 parser.parse_chunk("<tool_calls><tool_call>get_weather<tool_sep>").unwrap();
649
650 let error = parser.finish().unwrap_err();
@@ -575,7 +655,7 @@ mod tests {
655
656 #[test]
657 fn hy_v3_malformed_tool_call_fails_fast() {
578 - let mut parser = HyV3ToolParser::new(&test_tools());
658 + let mut parser = HyV3ToolParser::new(&test_tools(), "");
659 let error = parser
660 .parse_complete(
661 "<tool_calls><tool_call>get_weather<tool_sep><arg_key>city</arg_key><arg_value>Beijing</tool_call></tool_calls>",
rust/src/parser/src/tool/hy_v3/structural_tag.rs new
+261
@@ -0,0 +1,261 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +//! Structural-tag grammar for HY3 XML-style tool calls.
5 +
6 +use std::collections::HashSet;
7 +use std::sync::Arc;
8 +
9 +use serde_json::Value;
10 +use xgrammar_structural_tag::Result;
11 +use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagContext};
12 +use xgrammar_structural_tag::format::{Format, StructuralTag, TagFormat};
13 +use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam};
14 +
15 +use super::HyV3ToolMarkers;
16 +
17 +/// HY3 structural-tag builder using tokenizer-specific structural markers.
18 +#[derive(Debug, Clone)]
19 +pub(super) struct HyV3StructuralTagBuilder {
20 + markers: Arc<HyV3ToolMarkers>,
21 +}
22 +
23 +impl HyV3StructuralTagBuilder {
24 + pub(super) fn new(markers: Arc<HyV3ToolMarkers>) -> Self {
25 + Self { markers }
26 + }
27 +
28 + fn argument_pair(&self, key: &str) -> Format {
29 + let excludes = self.markers.iter().collect::<Vec<_>>();
30 + Format::sequence(vec![
31 + Format::const_string(&self.markers.arg_key_start),
32 + Format::const_string(key),
33 + Format::const_string(&self.markers.arg_key_end),
34 + Format::const_string("\n"),
35 + Format::const_string(&self.markers.arg_value_start),
36 + Format::any_text_excluding(&excludes),
37 + Format::const_string(&self.markers.arg_value_end),
38 + Format::const_string("\n"),
39 + ])
40 + }
41 +
42 + fn tool_call(&self, tool: &FunctionToolParam) -> TagFormat {
43 + let (required_keys, optional_keys) = argument_keys(tool.function.parameters.as_ref());
44 + let mut elements =
45 + required_keys.into_iter().map(|key| self.argument_pair(key)).collect::<Vec<_>>();
46 +
47 + if !optional_keys.is_empty() {
48 + let mut pairs =
49 + optional_keys.into_iter().map(|key| self.argument_pair(key)).collect::<Vec<_>>();
50 + let optional = if pairs.len() == 1 {
51 + pairs.pop().unwrap()
52 + } else {
53 + Format::or(pairs)
54 + };
55 + elements.push(Format::star(optional));
56 + }
57 +
58 + let content = if elements.is_empty() {
59 + Format::any_text()
60 + } else {
61 + Format::sequence(elements)
62 + };
63 + TagFormat::new(
64 + format!(
65 + "{}{}{}\n",
66 + self.markers.tool_call_start, tool.function.name, self.markers.tool_sep
67 + ),
68 + content,
69 + self.markers.tool_call_end.clone(),
70 + )
71 + }
72 +
73 + fn tool_calls(&self, tools: &[FunctionToolParam], choice: BuilderToolChoice) -> Format {
74 + let mut calls = tools.iter().map(|tool| self.tool_call(tool)).collect::<Vec<_>>();
75 + let begin = format!("{}\n", self.markers.tool_calls_start);
76 + let end = format!("\n{}", self.markers.tool_calls_end);
77 +
78 + match choice {
79 + BuilderToolChoice::Auto if calls.is_empty() => Format::any_text(),
80 + BuilderToolChoice::Auto => {
81 + let outer = TagFormat::new(
82 + begin,
83 + Format::tags_with_separator(calls, "\n", true, false),
84 + end,
85 + );
86 + Format::triggered_tags(&[&self.markers.tool_calls_start], vec![outer])
87 + }
88 + BuilderToolChoice::Forced => Format::sequence(vec![
89 + Format::const_string(begin),
90 + Format::Tag(calls.pop().unwrap()),
91 + Format::const_string(end),
92 + ]),
93 + BuilderToolChoice::Required => Format::sequence(vec![
94 + Format::const_string(begin),
95 + Format::tags_with_separator(calls, "\n", true, false),
96 + Format::const_string(end),
97 + ]),
98 + }
99 + }
100 +}
101 +
102 +impl StructuralTagBuilder for HyV3StructuralTagBuilder {
103 + fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
104 + Ok(StructuralTag::new(
105 + self.tool_calls(ctx.function_tools, ctx.tool_choice),
106 + ))
107 + }
108 +}
109 +
110 +/// Split argument keys into required and optional declaration-order groups.
111 +fn argument_keys(parameters: Option<&Value>) -> (Vec<&str>, Vec<&str>) {
112 + let Some(parameters) = parameters.and_then(Value::as_object) else {
113 + return (Vec::new(), Vec::new());
114 + };
115 + let properties = parameters.get("properties").and_then(Value::as_object);
116 + let required = parameters
117 + .get("required")
118 + .and_then(Value::as_array)
119 + .into_iter()
120 + .flatten()
121 + .filter_map(Value::as_str)
122 + .collect::<Vec<_>>();
123 + let required_set = required.iter().copied().collect::<HashSet<_>>();
124 +
125 + let mut required_keys = properties
126 + .into_iter()
127 + .flat_map(|properties| properties.keys())
128 + .map(String::as_str)
129 + .filter(|key| required_set.contains(key))
130 + .collect::<Vec<_>>();
131 + required_keys.extend(
132 + required
133 + .into_iter()
134 + .filter(|key| properties.is_none_or(|properties| !properties.contains_key(*key))),
135 + );
136 + let optional_keys = properties
137 + .into_iter()
138 + .flat_map(|properties| properties.keys())
139 + .map(String::as_str)
140 + .filter(|key| !required_set.contains(key))
141 + .collect();
142 + (required_keys, optional_keys)
143 +}
144 +
145 +#[cfg(test)]
146 +mod tests {
147 + use expect_test::expect;
148 + use serde_json::{Value, json};
149 + use std::sync::Arc;
150 + use xgrammar_structural_tag::builders::StructuralTagOptions;
151 + use xgrammar_structural_tag::{
152 + FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
153 + };
154 +
155 + use super::HyV3StructuralTagBuilder;
156 + use crate::tool::HyV3ToolMarkers;
157 +
158 + fn tool(name: &str, parameters: Value) -> ToolParam {
159 + ToolParam::Function(FunctionToolParam::new(
160 + FunctionDefinition::new(name).with_parameters(parameters),
161 + ))
162 + }
163 +
164 + fn build(
165 + suffix: &str,
166 + tools: &[ToolParam],
167 + choice: ToolChoice,
168 + ) -> xgrammar_structural_tag::format::StructuralTag {
169 + build_structural_tag(
170 + HyV3StructuralTagBuilder::new(Arc::new(HyV3ToolMarkers::new(suffix))),
171 + tools,
172 + choice,
173 + StructuralTagOptions::default().with_reasoning(false),
174 + )
175 + .unwrap()
176 + }
177 +
178 + #[test]
179 + fn required_uses_suffixed_hy3_skeleton_and_bounded_values() {
180 + let tag = build(
181 + ":opensource",
182 + &[tool(
183 + "get_weather",
184 + json!({
185 + "type": "object",
186 + "properties": {
187 + "city": { "type": "string" },
188 + "days": { "type": "integer" }
189 + },
190 + "required": ["city"]
191 + }),
192 + )],
193 + ToolChoice::required(),
194 + );
195 +
196 + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls:opensource>\n"},{"type":"tags_with_separator","tags":[{"begin":"<tool_call:opensource>get_weather<tool_sep:opensource>\n","content":{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"city"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]},{"type":"star","content":{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key:opensource>"},{"type":"const_string","value":"days"},{"type":"const_string","value":"</arg_key:opensource>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value:opensource>"},{"type":"any_text","excludes":["<tool_calls:opensource>","</tool_calls:opensource>","<tool_call:opensource>","</tool_call:opensource>","<tool_sep:opensource>","<arg_key:opensource>","</arg_key:opensource>","<arg_value:opensource>","</arg_value:opensource>"]},{"type":"const_string","value":"</arg_value:opensource>"},{"type":"const_string","value":"\n"}]}}]},"end":"</tool_call:opensource>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},{"type":"const_string","value":"\n</tool_calls:opensource>"}]}}"#]].assert_eq(&tag.to_json_string().unwrap());
197 + }
198 +
199 + #[test]
200 + fn optional_only_schema_keeps_declared_key_alternatives() {
201 + let tag = build(
202 + "",
203 + &[tool(
204 + "lookup",
205 + json!({
206 + "type": "object",
207 + "properties": {
208 + "query": { "type": "string" },
209 + "limit": { "type": "integer" }
210 + }
211 + }),
212 + )],
213 + ToolChoice::required(),
214 + );
215 + let value = serde_json::to_value(tag).unwrap();
216 +
217 + expect![[r#"{"type":"sequence","elements":[{"type":"star","content":{"type":"or","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"limit"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}}]}"#]].assert_eq(
218 + &serde_json::to_string(
219 + &value["format"]["elements"][1]["tags"][0]["content"],
220 + )
221 + .unwrap(),
222 + );
223 + }
224 +
225 + #[test]
226 + fn auto_and_forced_preserve_tool_choice_shape() {
227 + let tools = [
228 + tool("search", json!({ "type": "object" })),
229 + tool("lookup", json!({ "type": "object" })),
230 + ];
231 + let auto = build("", &tools, ToolChoice::auto()).to_json_string().unwrap();
232 + let forced = build("", &tools, ToolChoice::function("lookup")).to_json_string().unwrap();
233 +
234 + expect![[r#"{"type":"structural_tag","format":{"type":"triggered_tags","triggers":["<tool_calls>"],"tags":[{"begin":"<tool_calls>\n","content":{"type":"tags_with_separator","tags":[{"begin":"<tool_call>search<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"},{"begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"}],"separator":"\n","at_least_one":true,"stop_after_first":false},"end":"\n</tool_calls>"}],"at_least_one":false,"stop_after_first":false,"excludes":[]}}"#]].assert_eq(&auto);
235 + expect![[r#"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"const_string","value":"<tool_calls>\n"},{"type":"tag","begin":"<tool_call>lookup<tool_sep>\n","content":{"type":"any_text","excludes":[]},"end":"</tool_call>"},{"type":"const_string","value":"\n</tool_calls>"}]}}"#]].assert_eq(&forced);
236 + }
237 +
238 + #[test]
239 + fn missing_required_property_is_still_emitted() {
240 + let tag = build(
241 + "",
242 + &[tool(
243 + "search",
244 + json!({
245 + "type": "object",
246 + "properties": { "query": { "type": "string" } },
247 + "required": ["query", "tenant"]
248 + }),
249 + )],
250 + ToolChoice::required(),
251 + );
252 + let value = serde_json::to_value(tag).unwrap();
253 +
254 + expect![[r#"{"type":"sequence","elements":[{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"query"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]},{"type":"sequence","elements":[{"type":"const_string","value":"<arg_key>"},{"type":"const_string","value":"tenant"},{"type":"const_string","value":"</arg_key>"},{"type":"const_string","value":"\n"},{"type":"const_string","value":"<arg_value>"},{"type":"any_text","excludes":["<tool_calls>","</tool_calls>","<tool_call>","</tool_call>","<tool_sep>","<arg_key>","</arg_key>","<arg_value>","</arg_value>"]},{"type":"const_string","value":"</arg_value>"},{"type":"const_string","value":"\n"}]}]}"#]].assert_eq(
255 + &serde_json::to_string(
256 + &value["format"]["elements"][1]["tags"][0]["content"],
257 + )
258 + .unwrap(),
259 + );
260 + }
261 +}
rust/src/parser/src/tool/mod.rs
+1 -1
@@ -24,7 +24,7 @@ pub use deepseek_dsml::{DeepSeekV4ToolParser, DeepSeekV32ToolParser};
24 pub use deepseek_json::{DeepSeekV3ToolParser, DeepSeekV31ToolParser};
25 pub use error::{Result, ToolParserError};
26 pub use glm_xml::{Glm45MoeToolParser, Glm47MoeToolParser};
27 -pub use hy_v3::HyV3ToolParser;
27 +pub(crate) use hy_v3::{HyV3ToolMarkers, HyV3ToolParser};
28 pub use json::{
29 Granite4ToolParser, HermesToolParser, Internlm2ToolParser, Llama3JsonToolParser,
30 MistralToolParser, Phi4MiniJsonToolParser, Qwen3XmlToolParser,
rust/src/parser/src/unified/hy_v3.rs new
+207
@@ -0,0 +1,207 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +use vllm_tokenizer::{DynTokenizer, Tokenizer};
5 +
6 +use super::{CombinedParser, Result, UnifiedParser, UnifiedParserOutput, token_id};
7 +use crate::reasoning::HyV3ReasoningParser;
8 +use crate::tool::{HyV3ToolMarkers, HyV3ToolParser, StructuralTagBuilder, Tool};
9 +
10 +const HY_V3_MARKER_STEMS: &[&str] = &[
11 + "think",
12 + "tool_calls",
13 + "tool_call",
14 + "tool_sep",
15 + "arg_key",
16 + "arg_value",
17 +];
18 +
19 +/// Unified reasoning and tool parser for HY3 output.
20 +pub struct HyV3UnifiedParser {
21 + inner: CombinedParser,
22 +}
23 +
24 +impl HyV3UnifiedParser {
25 + /// Create a HY3 parser using the suffix encoded in tokenizer added tokens.
26 + pub fn new(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
27 + let suffix = detect_token_suffix(tokenizer.as_ref());
28 + let markers = HyV3ToolMarkers::new(&suffix);
29 + for marker in markers.iter() {
30 + token_id(tokenizer.as_ref(), marker)?;
31 + }
32 +
33 + let reasoning = HyV3ReasoningParser::new(tokenizer, &suffix)?;
34 + let tool = HyV3ToolParser::new(tools, &suffix);
35 + Ok(Self {
36 + inner: CombinedParser::new(Some(Box::new(reasoning)), Some(Box::new(tool))),
37 + })
38 + }
39 +}
40 +
41 +impl UnifiedParser for HyV3UnifiedParser {
42 + fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
43 + where
44 + Self: Sized + 'static,
45 + {
46 + Self::new(tools, tokenizer).map(|parser| Box::new(parser) as Box<dyn UnifiedParser>)
47 + }
48 +
49 + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
50 + self.inner.initialize(prompt_token_ids)
51 + }
52 +
53 + fn preserve_special_tokens(&self) -> bool {
54 + self.inner.preserve_special_tokens()
55 + }
56 +
57 + fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
58 + self.inner.structural_tag_builder()
59 + }
60 +
61 + fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
62 + self.inner.tool_call_id(tool_index)
63 + }
64 +
65 + fn parse_into(&mut self, delta: &str, output: &mut UnifiedParserOutput) -> Result<()> {
66 + self.inner.parse_into(delta, output)
67 + }
68 +
69 + fn finish(&mut self) -> Result<UnifiedParserOutput> {
70 + self.inner.finish()
71 + }
72 +
73 + fn reset(&mut self) -> String {
74 + self.inner.reset()
75 + }
76 +}
77 +
78 +/// Detect the HY3 structural-token suffix from tokenizer added vocabulary.
79 +fn detect_token_suffix(tokenizer: &dyn Tokenizer) -> String {
80 + tokenizer
81 + .added_vocab()
82 + .iter()
83 + .filter_map(|(token, id)| marker_suffix(token).map(|suffix| (*id, suffix)))
84 + .min_by_key(|(id, _)| *id)
85 + .map(|(_, suffix)| suffix.to_string())
86 + .unwrap_or_default()
87 +}
88 +
89 +/// Extract the suffix from one opening or closing HY3 structural token.
90 +fn marker_suffix(token: &str) -> Option<&str> {
91 + let body = token.strip_prefix('<')?.strip_suffix('>')?;
92 + let body = body.strip_prefix('/').unwrap_or(body);
93 +
94 + HY_V3_MARKER_STEMS.iter().find_map(|stem| {
95 + let suffix = body.strip_prefix(stem)?;
96 + (suffix.is_empty() || suffix.starts_with(':')).then_some(suffix)
97 + })
98 +}
99 +
100 +#[cfg(test)]
101 +mod tests {
102 + use std::sync::Arc;
103 +
104 + use serde_json::json;
105 + use vllm_tokenizer::{Tokenizer, test_utils::TestTokenizer};
106 + use xgrammar_structural_tag::builders::StructuralTagOptions;
107 + use xgrammar_structural_tag::{
108 + FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
109 + };
110 +
111 + use super::{HyV3UnifiedParser, UnifiedParser};
112 + use crate::tool::Tool;
113 + use crate::unified::{UnifiedParserEvent, UnifiedParserOutput};
114 +
115 + fn tokenizer() -> TestTokenizer {
116 + [
117 + "<think:opensource>",
118 + "</think:opensource>",
119 + "<tool_calls:opensource>",
120 + "</tool_calls:opensource>",
121 + "<tool_call:opensource>",
122 + "</tool_call:opensource>",
123 + "<tool_sep:opensource>",
124 + "<arg_key:opensource>",
125 + "</arg_key:opensource>",
126 + "<arg_value:opensource>",
127 + "</arg_value:opensource>",
128 + ]
129 + .into_iter()
130 + .enumerate()
131 + .fold(TestTokenizer::new(), |tokenizer, (index, token)| {
132 + tokenizer.with_regular_token(token, 1000 + index as u32)
133 + })
134 + }
135 +
136 + fn tools() -> Vec<Tool> {
137 + vec![Tool {
138 + name: "get_weather".to_string(),
139 + description: None,
140 + parameters: json!({
141 + "type": "object",
142 + "properties": { "city": { "type": "string" } },
143 + }),
144 + strict: None,
145 + }]
146 + }
147 +
148 + #[test]
149 + fn parses_suffixed_reasoning_and_tool_call_as_one_stream() {
150 + let tokenizer = Arc::new(tokenizer());
151 + let think_start_id = tokenizer.token_to_id("<think:opensource>").unwrap();
152 + let mut parser = HyV3UnifiedParser::new(&tools(), tokenizer).unwrap();
153 + parser.initialize(&[think_start_id]).unwrap();
154 +
155 + let chunks = [
156 + "reasoning</think:open",
157 + "source>answer<tool_calls:opensource><tool_call:opensource>get_weather",
158 + "<tool_sep:opensource><arg_key:opensource>city</arg_key:opensource>",
159 + "<arg_value:opensource>Beijing</arg_value:opensource>",
160 + "</tool_call:opensource></tool_calls:opensource>",
161 + ];
162 + let mut output = UnifiedParserOutput::default();
163 + for chunk in chunks {
164 + parser.parse_into(chunk, &mut output).unwrap();
165 + }
166 + output.append(parser.finish().unwrap());
167 +
168 + assert_eq!(
169 + output.events,
170 + vec![
171 + UnifiedParserEvent::Reasoning("reasoning".to_string()),
172 + UnifiedParserEvent::Text("answer".to_string()),
173 + UnifiedParserEvent::ToolCall(crate::tool::ToolCallDelta {
174 + tool_index: 0,
175 + name: Some("get_weather".to_string()),
176 + arguments: r#"{"city":"Beijing"}"#.to_string(),
177 + }),
178 + ]
179 + );
180 + }
181 +
182 + #[test]
183 + fn structural_tag_uses_tokenizer_detected_suffix() {
184 + let parser = HyV3UnifiedParser::new(&tools(), Arc::new(tokenizer())).unwrap();
185 + let structural_tools = [ToolParam::Function(FunctionToolParam::new(
186 + FunctionDefinition::new("get_weather").with_parameters(json!({
187 + "type": "object",
188 + "properties": { "city": { "type": "string" } },
189 + "required": ["city"]
190 + })),
191 + ))];
192 +
193 + let tag = build_structural_tag(
194 + parser.structural_tag_builder().unwrap(),
195 + &structural_tools,
196 + ToolChoice::required(),
197 + StructuralTagOptions::default().with_reasoning(false),
198 + )
199 + .unwrap()
200 + .to_json_string()
201 + .unwrap();
202 +
203 + assert!(tag.contains("<tool_calls:opensource>"));
204 + assert!(tag.contains("<arg_value:opensource>"));
205 + assert!(!tag.contains("glm_xml"));
206 + }
207 +}
rust/src/parser/src/unified/mod.rs
+2
@@ -5,11 +5,13 @@
5
6 mod combined;
7 mod gemma4;
8 +mod hy_v3;
9 mod inkling;
10 mod kimi_k3;
11
12 pub use combined::CombinedParser;
13 pub use gemma4::Gemma4UnifiedParser;
14 +pub use hy_v3::HyV3UnifiedParser;
15 pub use inkling::InklingUnifiedParser;
16 pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser};
17 use thiserror::Error;
rust/src/text/src/backend/hf/config.rs
+38 -1
@@ -111,12 +111,32 @@ pub(super) struct GenerationConfig {
111 pub eos_token_id: Option<OneOrManyTokenIds>,
112 pub temperature: Option<f32>,
113 pub top_p: Option<f32>,
114 + #[serde(deserialize_with = "deserialize_top_k")]
115 pub top_k: Option<u32>,
116 pub min_p: Option<f32>,
117 pub repetition_penalty: Option<f32>,
118 pub max_new_tokens: Option<u32>,
119 }
120
121 +/// Deserialize vLLM-compatible `top_k` values from generation configs.
122 +///
123 +/// Both `-1` and `0` disable top-k sampling and become `None`; positive values
124 +/// are preserved.
125 +fn deserialize_top_k<'de, D>(deserializer: D) -> std::result::Result<Option<u32>, D::Error>
126 +where
127 + D: serde::Deserializer<'de>,
128 +{
129 + match Option::<i64>::deserialize(deserializer)? {
130 + None | Some(-1) | Some(0) => Ok(None),
131 + Some(value) if value > 0 => {
132 + u32::try_from(value).map(Some).map_err(serde::de::Error::custom)
133 + }
134 + Some(value) => Err(serde::de::Error::custom(format!(
135 + "top_k must be -1, 0, or a positive integer, got {value}"
136 + ))),
137 + }
138 +}
139 +
140 /// HF generation configs allow either one EOS id or a list of EOS ids.
141 #[derive(Debug, Clone, Deserialize)]
142 #[serde(untagged)]
@@ -294,7 +314,24 @@ where
314
315 #[cfg(test)]
316 mod tests {
297 - use super::ModelConfig;
317 + use super::{GenerationConfig, ModelConfig};
318 +
319 + #[test]
320 + fn generation_config_normalizes_vllm_top_k_values() {
321 + for (input, expected) in [
322 + (r#"{"top_k":null}"#, None),
323 + (r#"{"top_k":-1}"#, None),
324 + (r#"{"top_k":0}"#, None),
325 + (r#"{"top_k":20}"#, Some(20)),
326 + ] {
327 + let config: GenerationConfig = serde_json::from_str(input).unwrap();
328 + assert_eq!(config.top_k, expected, "input={input}");
329 + }
330 +
331 + for input in [r#"{"top_k":-2}"#, r#"{"top_k":4294967296}"#] {
332 + assert!(serde_json::from_str::<GenerationConfig>(input).is_err());
333 + }
334 + }
335
336 #[test]
337 fn model_config_detects_moe_from_named_expert_fields() {
rust/src/tokenizer/src/hf.rs
+33
@@ -149,10 +149,20 @@ fn encode_fastokens_ordinary(
149 pub struct HuggingFaceTokenizer {
150 backend: Backend,
151 special_token_ids: Arc<[u32]>,
152 + added_vocab: Box<[(String, u32)]>,
153 }
154
155 impl HuggingFaceTokenizer {
156 fn from_hf_backend(tokenizer: HfTokenizer) -> Self {
157 + let added_vocab = {
158 + let mut vocab: Vec<_> = tokenizer
159 + .get_added_tokens_decoder()
160 + .iter()
161 + .map(|(&id, token)| (token.content.clone(), id))
162 + .collect();
163 + vocab.sort_unstable_by_key(|(_, id)| *id);
164 + vocab.into_boxed_slice()
165 + };
166 let special_token_ids = {
167 let mut ids: Vec<u32> = tokenizer
168 .get_added_tokens_decoder()
@@ -167,10 +177,21 @@ impl HuggingFaceTokenizer {
177 Self {
178 backend: Backend::Hf(Box::new(tokenizer)),
179 special_token_ids,
180 + added_vocab,
181 }
182 }
183
184 fn from_fastokens_backend(tokenizer: FastokensTokenizer) -> Self {
185 + let added_vocab = {
186 + let mut vocab: Vec<_> = tokenizer
187 + .added_tokens()
188 + .into_iter()
189 + .flat_map(|added_tokens| added_tokens.iter())
190 + .map(|token| (token.content.to_string(), token.id))
191 + .collect();
192 + vocab.sort_unstable_by_key(|(_, id)| *id);
193 + vocab.into_boxed_slice()
194 + };
195 let special_token_ids = {
196 let mut ids: Vec<u32> = tokenizer
197 .added_tokens()
@@ -192,6 +213,7 @@ impl HuggingFaceTokenizer {
213 Self {
214 backend,
215 special_token_ids,
216 + added_vocab,
217 }
218 }
219
@@ -292,6 +314,10 @@ impl Tokenizer for HuggingFaceTokenizer {
314 }
315 }
316
317 + fn added_vocab(&self) -> &[(String, u32)] {
318 + &self.added_vocab
319 + }
320 +
321 fn is_special_id(&self, token_id: u32) -> bool {
322 self.special_token_ids.binary_search(&token_id).is_ok()
323 }
@@ -477,6 +503,13 @@ mod tests {
503 tokenizer.encode(SPECIAL_TOKEN, false).unwrap(),
504 vec![tokenizer.token_to_id(SPECIAL_TOKEN).unwrap()]
505 );
506 + assert_eq!(
507 + tokenizer.added_vocab(),
508 + vec![
509 + (REGULAR_TOKEN.to_string(), 256),
510 + (SPECIAL_TOKEN.to_string(), 257),
511 + ]
512 + );
513
514 for text in [
515 "",
rust/src/tokenizer/src/lib.rs
+9
@@ -39,6 +39,15 @@ pub trait Tokenizer: Send + Sync {
39 /// Convert one token ID into the tokenizer's raw token string.
40 fn id_to_token(&self, id: u32) -> Option<String>;
41
42 + /// Borrow tokenizer added-vocabulary entries as `(token, id)` pairs.
43 + ///
44 + /// Backends that do not expose the distinction between model vocabulary
45 + /// and added vocabulary return an empty list.
46 + // TODO: add support to all tokenizer backends
47 + fn added_vocab(&self) -> &[(String, u32)] {
48 + &[]
49 + }
50 +
51 /// Return the vocabulary size. Backends that cannot report it fall back to
52 /// `usize::MAX`, an effectively unbounded value used only by test stubs.
53 fn vocab_size(&self) -> usize {
rust/src/tokenizer/src/test_utils.rs
+19 -1
@@ -69,6 +69,7 @@ struct TestToken {
69 pub struct TestTokenizer {
70 token_to_id: BTreeMap<String, u32>,
71 id_to_token: BTreeMap<u32, TestToken>,
72 + added_vocab: Vec<(String, u32)>,
73 unknown_decode: UnknownDecode,
74 vocab_size: Option<usize>,
75 bos_token_id: Option<u32>,
@@ -86,6 +87,7 @@ impl TestTokenizer {
87 Self {
88 token_to_id: BTreeMap::new(),
89 id_to_token: BTreeMap::new(),
90 + added_vocab: Vec::new(),
91 unknown_decode: UnknownDecode::Error,
92 vocab_size: None,
93 bos_token_id: None,
@@ -154,9 +156,21 @@ impl TestTokenizer {
156 if self.token_to_id.insert(token.clone(), id).is_some() {
157 panic!("configured test token text {token:?} was registered more than once");
158 }
157 - if self.id_to_token.insert(id, TestToken { text: token, kind }).is_some() {
159 + if self
160 + .id_to_token
161 + .insert(
162 + id,
163 + TestToken {
164 + text: token.clone(),
165 + kind,
166 + },
167 + )
168 + .is_some()
169 + {
170 panic!("configured test token id {id} was registered more than once");
171 }
172 + self.added_vocab.push((token, id));
173 + self.added_vocab.sort_unstable_by_key(|(_, id)| *id);
174 }
175
176 fn byte_to_token(id: u32) -> Option<String> {
@@ -254,6 +268,10 @@ impl Tokenizer for TestTokenizer {
268 .or_else(|| Self::byte_to_token(id))
269 }
270
271 + fn added_vocab(&self) -> &[(String, u32)] {
272 + &self.added_vocab
273 + }
274 +
275 fn vocab_size(&self) -> usize {
276 self.vocab_size.unwrap_or_else(|| self.inferred_vocab_size())
277 }