@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 24fb5fc08c

[Rust Frontend] Add `hf` parser for response templates (#59005)

Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Bugen Zhao committed Oct 1, 2026 at 16:16 UTC 24fb5fc08c2bcae8575bd2e861f463d8cf919ffb

23 files changed +4573 -33

rust/Cargo.lock

@@ -5971,12 +5971,16 @@ dependencies = [

5971 "expect-test",

5972 "futures",

5973 "openai-protocol",

5974 + "regex-automata",

5975 + "regex-syntax",

5976 "serde",

5977 "serde_json",

5978 + "serde_path_to_error",

5979 "serde_with",

5980 "thiserror 2.0.18",

5981 "thiserror-ext",

5982 "tool-parser",

5983 + "tracing",

5984 "vllm-tokenizer",

5985 "winnow",

5986 "xgrammar-structural-tag",

rust/Cargo.toml

@@ -85,6 +85,8 @@ rand = "0.9.3"

85 rand_distr = "0.5.1"

86 rayon = "1.11.0"

87 reasoning-parser = "1.2.2"

88 +regex-automata = "0.4.14"

89 +regex-syntax = "0.8.10"

90 reqwest = { version = "0.12.8", default-features = false, features = ["native-tls"] }

91 reqwest-0-13 = { package = "reqwest", version = "0.13.4", default-features = false, features = ["native-tls"] }

92 riptoken = { version = "0.3.0", default-features = false }

@@ -96,6 +98,7 @@ serde = { version = "1.0.228", features = ["derive", "rc"] }

98 serde-json-fmt = "0.1.0"

99 serde_default = "0.2.0"

100 serde_json = { version = "1.0.145", features = ["preserve_order"] }

101 +serde_path_to_error = "0.1.20"

102 serde_repr = "0.1.20"

103 serde_tuple = "1.1.3"

104 serde_with = "3.21.0"

rust/src/chat/src/backend/hf.rs

+46 -10

@@ -3,8 +3,11 @@

4 use std::sync::Arc;

6 -use tracing::info;

7 -use vllm_text::backend::hf::{HfTextBackend, ResolvedModelFiles, load_model_config};

6 +use thiserror_ext::AsReport as _;

7 +use tracing::{info, warn};

8 +use vllm_text::backend::hf::{

9 + HfTextBackend, ResolvedModelFiles, load_model_config, load_tokenizer_config,

10 +};

11 use vllm_text::tokenizer::DynTokenizer;

12 use vllm_text::{DynTextBackend, TextBackend as _};

@@ -17,6 +20,7 @@ use crate::multimodal::{MultimodalConfigFiles, MultimodalModelInfo};

20 use crate::output::{

21 DefaultChatOutputProcessor, HarmonyChatOutputProcessor, validate_harmony_parser_overrides,

22 };

23 +use crate::parser::unified::{HfTemplateError, ResponseTemplate};

24 use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};

25 use crate::renderer::{

26 DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer, DynChatRenderer,

@@ -32,6 +36,9 @@ pub struct HfChatBackend {

36 tokenizer: DynTokenizer,

37 chat_renderer: DynChatRenderer,

38 multimodal_model_info: Option<MultimodalModelInfo>,

39 + /// `response_template` from `tokenizer_config.json` for the `hf` parser, or

40 + /// why it is unavailable.

41 + response_template: std::result::Result<Arc<ResponseTemplate>, HfTemplateError>,

42 }

44 impl HfChatBackend {

@@ -61,6 +68,7 @@ impl HfChatBackend {

68 )?

69 };

70 let multimodal_render_info = resolve_multimodal_render_info(multimodal_model_info.as_ref());

71 + let response_template = load_response_template(&files)?;

73 let renderer = options.renderer.resolve(model_type);

74 let chat_renderer: DynChatRenderer = match renderer {

@@ -105,10 +113,35 @@ impl HfChatBackend {

113 tokenizer,

114 chat_renderer,

115 multimodal_model_info,

116 + response_template,

117 })

118 }

119 }

120

121 +/// Load and compile the checkpoint's `response_template`, if any.

122 +///

123 +/// A missing or unusable template is kept as an error, so that only requests

124 +/// selecting the `hf` parser fail.

125 +fn load_response_template(

126 + files: &ResolvedModelFiles,

127 +) -> Result<std::result::Result<Arc<ResponseTemplate>, HfTemplateError>> {

128 + let Some(template) =

129 + load_tokenizer_config(files.tokenizer_config_path.as_deref())?.response_template

130 + else {

131 + return Ok(Err(HfTemplateError::Missing));

132 + };

133 + Ok(match ResponseTemplate::from_json(&template) {

134 + Ok(template) => {

135 + info!("loaded response_template for the `hf` parser");

136 + Ok(Arc::new(template))

137 + }

138 + Err(error) => {

139 + warn!(error = %error.as_report(), "the `hf` parser cannot execute the model's response_template");

140 + Err(error)

141 + }

142 + })

143 +}

144 +

145 impl ChatBackend for HfChatBackend {

146 fn chat_renderer(&self) -> DynChatRenderer {

147 self.chat_renderer.clone()

@@ -128,14 +161,17 @@ impl ChatBackend for HfChatBackend {

161 return Ok(Box::new(HarmonyChatOutputProcessor::new(request)?));

162 }

163

131 - Ok(Box::new(DefaultChatOutputProcessor::new(

132 - request,

133 - &self.model_id,

134 - self.tokenizer.clone(),

135 - options.tool_call_parser,

136 - options.reasoning_parser,

137 - options.tool_strict_level,

138 - )?))

164 + Ok(Box::new(

165 + DefaultChatOutputProcessor::with_response_template(

166 + request,

167 + &self.model_id,

168 + self.tokenizer.clone(),

169 + &self.response_template,

170 + options.tool_call_parser,

171 + options.reasoning_parser,

172 + options.tool_strict_level,

173 + )?,

174 + ))

175 }

176 }

177

rust/src/chat/src/lib.rs

+8 -2

@@ -412,6 +412,12 @@ mod tests {

412 validate_parser_overrides(&selection, &selection).unwrap();

413 }

414

415 + #[test]

416 + fn validate_parser_overrides_accepts_explicit_hf() {

417 + let selection = ParserSelection::Explicit("hf".to_string());

418 + validate_parser_overrides(&selection, &selection).unwrap();

419 + }

420 +

421 #[test]

422 fn validate_parser_overrides_accepts_auto_and_none() {

423 validate_parser_overrides(&ParserSelection::Auto, &ParserSelection::None).unwrap();

@@ -425,7 +431,7 @@ mod tests {

431 )

432 .unwrap_err();

433

428 - expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());

434 + expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, granite4, hermes, hf, hy_v3, hy_v4, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, mimo, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());

435 }

436

437 #[test]

@@ -436,6 +442,6 @@ mod tests {

442 )

443 .unwrap_err();

444

439 - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());

445 + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, deepseek_v41, gemma4, glm45, glm47, hf, hy_v3, hy_v4, inkling, kimi, kimi_k2, kimi_k3, mimo, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());

446 }

447 }

rust/src/chat/src/output/default/mod.rs

+145 -5

@@ -5,7 +5,7 @@

6 mod unified;

8 -use std::sync::Once;

8 +use std::sync::{Arc, Once};

10 use futures::StreamExt as _;

11 use tracing::info;

@@ -20,7 +20,9 @@ use crate::error::Result;

20 use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream};

21 use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory};

22 use crate::parser::tool::{ToolParser, ToolParserFactory};

23 -use crate::parser::unified::UnifiedParserFactory;

23 +use crate::parser::unified::{

24 + HfTemplateError, HfUnifiedParser, ResponseTemplate, UnifiedParserFactory, names,

25 +};

26 use crate::parser::{ParserSelection, ToolStrictLevel};

27 use crate::request::{ChatRequest, ChatTool};

28 use crate::{Error, Result as ChatResult};

@@ -64,11 +66,48 @@ impl DefaultChatOutputProcessor {

66 reasoning_parser: &ParserSelection,

67 tool_strict_level: ToolStrictLevel,

68 ) -> ChatResult<Self> {

69 + Self::with_response_template(

70 + request,

71 + model_id,

72 + tokenizer,

73 + &Err(HfTemplateError::Missing),

74 + tool_call_parser,

75 + reasoning_parser,

76 + tool_strict_level,

77 + )

78 + }

79 +

80 + /// Like [`Self::new`], additionally providing the model's `response_template`

81 + /// for the `hf` parser, or why it is unavailable.

82 + pub fn with_response_template(

83 + request: &mut ChatRequest,

84 + model_id: &str,

85 + tokenizer: DynTokenizer,

86 + response_template: &std::result::Result<Arc<ResponseTemplate>, HfTemplateError>,

87 + tool_call_parser: &ParserSelection,

88 + reasoning_parser: &ParserSelection,

89 + tool_strict_level: ToolStrictLevel,

90 + ) -> ChatResult<Self> {

91 + let tool_name = tool_call_parser.resolve_tool_name(model_id);

92 + let reasoning_name = reasoning_parser.resolve_reasoning_name(model_id);

93 + // `hf` is opt-in: `Auto` never selects it, but a usable template is worth a hint.

94 + let auto = [tool_call_parser, reasoning_parser]

95 + .iter()

96 + .any(|selection| matches!(selection, ParserSelection::Auto));

97 + if auto && tool_name.is_none() && reasoning_name.is_none() && response_template.is_ok() {

98 + RESPONSE_TEMPLATE_HINT_ONCE.call_once(|| {

99 + info!(

100 + "the model ships a response_template; pass `--tool-call-parser hf \

101 + --reasoning-parser hf` to parse with it"

102 + );

103 + });

104 + }

105 let parser = if let Some(parser) = Self::resolve_optional_unified_parser(

106 request.tools(),

107 tokenizer.clone(),

70 - tool_call_parser.resolve_tool_name(model_id),

71 - reasoning_parser.resolve_reasoning_name(model_id),

108 + response_template,

109 + tool_name,

110 + reasoning_name,

111 )? {

112 parser

113 } else {

@@ -141,6 +180,7 @@ impl DefaultChatOutputProcessor {

180 fn resolve_optional_unified_parser(

181 tools: &[ChatTool],

182 tokenizer: DynTokenizer,

183 + response_template: &std::result::Result<Arc<ResponseTemplate>, HfTemplateError>,

184 tool_name: Option<&str>,

185 reasoning_name: Option<&str>,

186 ) -> ChatResult<Option<Box<dyn UnifiedParser>>> {

@@ -157,7 +197,18 @@ impl DefaultChatOutputProcessor {

197 });

198 }

199

160 - let parser = factory.create(parser_name, tools, tokenizer)?;

200 + // `hf` is built from the model's template, never by its registry constructor.

201 + let parser = if parser_name == names::HF {

202 + let template =

203 + response_template.as_ref().map_err(|error| Error::ParserInitialization {

204 + kind: "unified",

205 + name: parser_name.to_string(),

206 + error: Box::new(error.clone()),

207 + })?;

208 + Box::new(HfUnifiedParser::new(template.clone(), tools, tokenizer))

209 + } else {

210 + factory.create(parser_name, tools, tokenizer)?

211 + };

212

213 UNIFIED_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using unified parser"));

214 Ok(Some(parser))

@@ -186,6 +237,7 @@ impl DefaultChatOutputProcessor {

237 static TOOL_PARSER_LOG_ONCE: Once = Once::new();

238 static REASONING_PARSER_LOG_ONCE: Once = Once::new();

239 static UNIFIED_PARSER_LOG_ONCE: Once = Once::new();

240 +static RESPONSE_TEMPLATE_HINT_ONCE: Once = Once::new();

241

242 impl ChatOutputProcessor for DefaultChatOutputProcessor {

243 fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {

@@ -230,10 +282,12 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor {

282 mod tests {

283 use std::sync::Arc;

284

285 + use thiserror_ext::AsReport as _;

286 use vllm_tokenizer::test_utils::TestTokenizer;

287

288 use super::DefaultChatOutputProcessor;

289 use crate::output::ChatOutputProcessor;

290 + use crate::parser::unified::{HfTemplateError, ResponseTemplate};

291 use crate::parser::{ParserSelection, ToolStrictLevel};

292 use crate::request::{ChatRequest, ChatTool, ChatToolChoice, ResolvedToolContext};

293

@@ -390,4 +444,90 @@ mod tests {

444 expect_test::expect!["unified parsing requires the tool and reasoning selections to resolve to the same parser; resolved tool=qwen3_xml, reasoning=gemma4"]

445 .assert_eq(&format!("{error}"));

446 }

447 +

448 + type TemplateResult = Result<Arc<ResponseTemplate>, HfTemplateError>;

449 +

450 + fn loaded_template() -> TemplateResult {

451 + Ok(Arc::new(

452 + ResponseTemplate::from_json(&serde_json::json!({

453 + "start_anchor": "<|turn>model\n",

454 + "fields": {

455 + "thinking": {"open": "<|channel>thought\n", "close": "<channel|>"},

456 + "content": {"close": "<turn|>"},

457 + },

458 + }))

459 + .unwrap(),

460 + ))

461 + }

462 +

463 + fn invalid_template() -> TemplateResult {

464 + let error = ResponseTemplate::from_json(&serde_json::json!({"fields": {"content": {}}}))

465 + .unwrap_err();

466 + Err(error)

467 + }

468 +

469 + /// Build a processor for `model_id`, returning whether it keeps special

470 + /// tokens (only parsers that need them, such as `hf`, turn this on).

471 + fn keeps_special_tokens(

472 + model_id: &str,

473 + template: &TemplateResult,

474 + tool: &ParserSelection,

475 + reasoning: &ParserSelection,

476 + ) -> crate::Result<bool> {

477 + let mut request = ChatRequest::for_test();

478 + DefaultChatOutputProcessor::with_response_template(

479 + &mut request,

480 + model_id,

481 + tokenizer(),

482 + template,

483 + tool,

484 + reasoning,

485 + ToolStrictLevel::Auto,

486 + )?;

487 + Ok(!request.decode_options.skip_special_tokens)

488 + }

489 +

490 + #[test]

491 + fn explicit_hf_uses_the_model_response_template() {

492 + let hf = ParserSelection::Explicit("hf".to_string());

493 + assert!(keeps_special_tokens("other-model", &loaded_template(), &hf, &hf).unwrap());

494 + }

495 +

496 + #[test]

497 + fn explicit_hf_without_usable_template_fails() {

498 + let hf = ParserSelection::Explicit("hf".to_string());

499 + let error = |template| {

500 + keeps_special_tokens("other-model", &template, &hf, &hf)

501 + .unwrap_err()

502 + .to_report_string()

503 + };

504 + expect_test::expect!["failed to initialize unified parser `hf`: the model's tokenizer_config.json provides no response_template"]

505 + .assert_eq(&error(Err(HfTemplateError::Missing)));

506 + expect_test::expect!["failed to initialize unified parser `hf`: invalid response_template: response_template must define 'start_anchor' or 'start_anchor_pattern'."]

507 + .assert_eq(&error(invalid_template()));

508 + }

509 +

510 + #[test]

511 + fn auto_never_selects_hf() {

512 + let auto = ParserSelection::Auto;

513 + // Without a matching parser, parsing stays disabled even with a usable template.

514 + assert!(!keeps_special_tokens("other-model", &loaded_template(), &auto, &auto).unwrap());

515 + // An unusable template is not consulted unless `hf` is selected.

516 + assert!(!keeps_special_tokens("other-model", &invalid_template(), &auto, &auto).unwrap());

517 + keeps_special_tokens("google/gemma-4-27b-it", &invalid_template(), &auto, &auto).unwrap();

518 + }

519 +

520 + #[test]

521 + fn hf_requires_matching_parser_selections() {

522 + let error = keeps_special_tokens(

523 + "other-model",

524 + &loaded_template(),

525 + &ParserSelection::Explicit("hf".to_string()),

526 + &ParserSelection::None,

527 + )

528 + .unwrap_err();

529 +

530 + expect_test::expect!["unified parsing requires the tool and reasoning selections to resolve to the same parser; resolved tool=hf, reasoning=none"]

531 + .assert_eq(&format!("{error}"));

532 + }

533 }

rust/src/chat/src/parser/unified.rs

+9 -2

@@ -6,8 +6,8 @@

6 use std::sync::LazyLock;

8 pub use vllm_parser::unified::{

9 - Gemma4UnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser, InklingUnifiedParser,

10 - KimiK3UnifiedParser, UnifiedParser,

9 + Gemma4UnifiedParser, HfTemplateError, HfUnifiedParser, HyV3UnifiedParser, HyV4UnifiedParser,

10 + InklingUnifiedParser, KimiK3UnifiedParser, ResponseTemplate, UnifiedParser,

11 };

12 use vllm_tokenizer::DynTokenizer;

@@ -21,6 +21,11 @@ pub mod names {

21 pub const HY_V4: &str = "hy_v4";

22 pub const INKLING: &str = "inkling";

23 pub const KIMI_K3: &str = "kimi_k3";

24 +

25 + /// Executes the checkpoint's Hugging Face `response_template`. Registered for

26 + /// name resolution only: it is built from the model's template, never by its

27 + /// registry constructor.

28 + pub const HF: &str = "hf";

29 }

31 /// Constructor signature for one registered unified parser implementation.

@@ -48,6 +53,8 @@ impl UnifiedParserFactory {

53 factory.register_parser::<HyV4UnifiedParser>(names::HY_V4);

54 factory.register_parser::<InklingUnifiedParser>(names::INKLING);

55 factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);

56 + // Opt-in only: no model patterns map to it.

57 + factory.register_parser::<HfUnifiedParser>(names::HF);

59 factory

60 .register_pattern("gemma-4", names::GEMMA4)

rust/src/chat/tests/roundtrip.rs

+10

@@ -309,6 +309,15 @@ impl RoundtripCase {

309 }

310 }

311

312 + /// Gemma4 through the checkpoint's `response_template` (`hf` parser).

313 + fn gemma4_hf() -> Self {

314 + Self {

315 + tool_call_parser: ParserSelection::Explicit("hf".to_string()),

316 + reasoning_parser: ParserSelection::Explicit("hf".to_string()),

317 + ..Self::gemma4()

318 + }

319 + }

320 +

321 /// Kimi K2.5 tool-call format with `<think>` reasoning tags.

322 #[allow(dead_code)]

323 fn kimi_k25() -> Self {

@@ -459,6 +468,7 @@ roundtrip_tests! {

468 step3p5 => [reasoning_and_content],

469 nemotron_v3 => [reasoning_and_content],

470 gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call

471 + gemma4_hf => [tool_call_mix],

472 kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history

473 // K3 drops plain-assistant reasoning in history; tool-call turns keep it.

474 kimi_k3 => [tool_call_mix],

rust/src/parser/Cargo.toml

@@ -11,11 +11,15 @@ test-util = []

11 auto_impl.workspace = true

12 easy-ext.workspace = true

13 educe.workspace = true

14 +regex-automata.workspace = true

15 +regex-syntax.workspace = true

16 serde.workspace = true

17 serde_json.workspace = true

18 +serde_path_to_error.workspace = true

19 serde_with.workspace = true

20 thiserror.workspace = true

21 thiserror-ext.workspace = true

22 +tracing.workspace = true

23 vllm-tokenizer.workspace = true

24 winnow.workspace = true

25 xgrammar-structural-tag.workspace = true

rust/src/parser/src/tool/mod.rs

@@ -34,6 +34,7 @@ pub use kimi_k2::KimiK2ToolParser;

34 pub use mimo::MiMoToolParser;

35 pub use minimax_m2::MinimaxM2ToolParser;

36 pub use minimax_m3::MinimaxM3ToolParser;

37 +pub(crate) use parameters::{ToolSchemas, convert_integer_text, convert_number_text};

38 pub use qwen_coder::Qwen3CoderToolParser;

39 pub use seed_oss::SeedOssToolParser;

40 use serde::{Deserialize, Serialize};

rust/src/parser/src/tool/parameters.rs

+21 -11

@@ -9,7 +9,7 @@ use crate::tool::Tool;

10 /// Normalized parameter schemas for all tools in one request.

11 #[derive(Debug, Clone, Default, PartialEq, Eq)]

12 -pub(super) struct ToolSchemas {

12 +pub(crate) struct ToolSchemas {

13 tools: BTreeMap<String, ToolSchema>,

14 }

@@ -28,7 +28,7 @@ pub(super) struct ToolSchema {

28 ///

29 /// It can be either a raw text string, or a structured input with named child elements.

30 #[derive(Debug, Clone, PartialEq, Eq)]

31 -pub(super) enum ParamInput {

31 +pub(crate) enum ParamInput {

32 Text(String),

33 #[allow(dead_code)]

34 Elements(Vec<ParamElement>),

@@ -42,7 +42,7 @@ impl From<String> for ParamInput {

43 /// One named structured parameter child.

44 #[derive(Debug, Clone, PartialEq, Eq)]

45 -pub(super) struct ParamElement {

45 +pub(crate) struct ParamElement {

46 pub name: String,

47 pub value: ParamInput,

48 }

@@ -67,7 +67,7 @@ pub(super) enum JsonParamType {

68 impl ToolSchemas {

69 /// Normalize OpenAI-style tool parameter JSON schemas for one request.

70 - pub(super) fn from_tools(tools: &[Tool]) -> Self {

70 + pub(crate) fn from_tools(tools: &[Tool]) -> Self {

71 let tools = tools

72 .iter()

73 .map(|tool| (tool.name.clone(), ToolSchema::from_schema(&tool.parameters)))

@@ -98,7 +98,7 @@ impl ToolSchemas {

98 }

100 /// Convert one parameter value for one named tool.

101 - pub(super) fn convert_param_with_schema<P>(

101 + pub(crate) fn convert_param_with_schema<P>(

102 &self,

103 function_name: &str,

104 name: &str,

@@ -361,12 +361,8 @@ fn try_convert_value(param_type: &JsonParamType, input: &ParamInput) -> Option<V

361 fn try_convert_text_value(param_type: &JsonParamType, value: &str) -> Option<Value> {

362 match param_type {

363 JsonParamType::String => Some(Value::String(value.to_string())),

364 - JsonParamType::Integer => without_digit_separators(value.trim())?

365 - .parse::<i64>()

366 - .ok()

367 - .map(Number::from)

368 - .map(Value::Number),

369 - JsonParamType::Number => try_convert_number(&without_digit_separators(value.trim())?),

364 + JsonParamType::Integer => convert_integer_text(value),

365 + JsonParamType::Number => convert_number_text(value),

366 JsonParamType::Boolean => try_convert_boolean(value),

367 JsonParamType::Object { .. } if value.trim().is_empty() => Some(Value::Object(Map::new())),

368 JsonParamType::Array { .. } if value.trim().is_empty() => Some(Value::Array(Vec::new())),

@@ -461,6 +457,20 @@ fn without_digit_separators(value: &str) -> Option<std::borrow::Cow<'_, str>> {

457 valid.then(|| value.replace('_', "").into())

458 }

459

460 +/// Convert raw text to a JSON integer, as for an `integer` parameter.

461 +pub(crate) fn convert_integer_text(value: &str) -> Option<Value> {

462 + without_digit_separators(value.trim())?

463 + .parse::<i64>()

464 + .ok()

465 + .map(Number::from)

466 + .map(Value::Number)

467 +}

468 +

469 +/// Convert raw text to a JSON number, as for a `number` parameter.

470 +pub(crate) fn convert_number_text(value: &str) -> Option<Value> {

471 + try_convert_number(&without_digit_separators(value.trim())?)

472 +}

473 +

474 /// Convert one raw string value to a JSON number.

475 fn try_convert_number(value: &str) -> Option<Value> {

476 serde_json::from_str::<Number>(value)

rust/src/parser/src/unified/combined.rs

+4 -1

@@ -65,7 +65,10 @@ impl UnifiedParser for CombinedParser {

65 where

66 Self: Sized + 'static,

67 {

68 - Err(UnifiedParserError::CombinedParserConstructor)

68 + Err(UnifiedParserError::NoNamedConstructor {

69 + parser: "combined",

70 + built_from: "split parser instances",

71 + })

72 }

74 fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {

rust/src/parser/src/unified/hf/content.rs new

+537

@@ -0,0 +1,537 @@

1 +// SPDX-License-Identifier: Apache-2.0

2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +//! Content parsers: turn one region's captured text into a JSON value.

5 +//!

6 +//! Port of `content_parsers.py`. Each parser takes a chunk of captured text and

7 +//! parses it into a single key in the output message dictionary.

8 +

9 +use regex_automata::meta::Regex;

10 +use regex_automata::util::syntax;

11 +use serde::Deserialize;

12 +use serde_json::{Map, Value};

13 +use thiserror_ext::AsReport as _;

14 +

15 +use super::spec::{ContentKind, deserialize};

16 +use super::{Result, invalid, unsupported};

17 +use crate::tool::{convert_integer_text, convert_number_text};

18 +use crate::unified::{self, parsing_failed};

19 +

20 +// Sentinel characters for lax-JSON string pre-extraction — ASCII control chars

21 +// that should never appear in real LLM output.

22 +const LAX_OPEN: char = '\u{1}';

23 +const LAX_CLOSE: char = '\u{2}';

24 +

25 +/// A configured content parser (`content` plus `content_args` of one field).

26 +#[derive(Debug, Clone)]

27 +pub(super) enum ContentParser {

28 + Text(TextArgs),

29 + /// Converted like an `integer` tool parameter.

30 + Int,

31 + /// Converted like a `number` tool parameter.

32 + Float,

33 + Bool(TextArgs),

34 + Json(JsonArgs),

35 + XmlInline(XmlInlineArgs),

36 + KvLines(KvLinesArgs),

37 +}

38 +

39 +/// The `strip` argument shared by parsers that go through `_text`.

40 +#[derive(Debug, Clone, Copy, Deserialize)]

41 +pub(super) struct TextArgs {

42 + #[serde(default = "default_strip")]

43 + pub strip: bool,

44 +}

45 +

46 +#[derive(Debug, Clone, Deserialize)]

47 +pub(super) struct JsonArgs {

48 + #[serde(flatten)]

49 + text: TextArgs,

50 + /// Quote bare-identifier keys before parsing.

51 + #[serde(default)]

52 + unquoted_keys: bool,

53 + /// Strings delimited by these custom markers are pre-extracted, then restored

54 + /// as standard JSON strings.

55 + #[serde(default)]

56 + string_delims: Vec<(String, String)>,

57 + /// Return stripped text if parsing fails.

58 + #[serde(default)]

59 + allow_non_json: bool,

60 +}

61 +

62 +#[derive(Debug, Clone)]

63 +pub(super) struct XmlInlineArgs {

64 + tag_pattern: Regex,

65 + /// Index of the required `key` group.

66 + key_group: usize,

67 + /// Index of the `value` group, if the pattern declares one.

68 + value_group: Option<usize>,

69 + value_parser: Option<Box<ContentParser>>,

70 + /// Collect duplicate keys into a list.

71 + merge_duplicates: bool,

72 +}

73 +

74 +#[derive(Debug, Clone)]

75 +pub(super) struct KvLinesArgs {

76 + text: TextArgs,

77 + line_sep: String,

78 + kv_sep: String,

79 + value_parser: Option<Box<ContentParser>>,

80 +}

81 +

82 +/// `content_args` of `xml-inline`.

83 +#[derive(Deserialize)]

84 +struct XmlInlineSpec {

85 + /// Must have named groups `key` and `value`.

86 + tag_pattern: String,

87 + value_parser: Option<ValueParserSpec>,

88 + #[serde(default)]

89 + merge_duplicates: bool,

90 +}

91 +

92 +/// `content_args` of `kv-lines`.

93 +#[derive(Deserialize)]

94 +struct KvLinesSpec {

95 + #[serde(flatten)]

96 + text: TextArgs,

97 + #[serde(default = "default_line_sep")]

98 + line_sep: String,

99 + #[serde(default = "default_kv_sep")]

100 + kv_sep: String,

101 + value_parser: Option<ValueParserSpec>,

102 +}

103 +

104 +/// A nested `value_parser`: `{"name": ..., "args": {...}}`.

105 +#[derive(Deserialize)]

106 +struct ValueParserSpec {

107 + #[serde(default)]

108 + name: ContentKind,

109 + #[serde(default)]

110 + args: Map<String, Value>,

111 +}

112 +

113 +impl ContentParser {

114 + /// Build a parser from its kind and `content_args`.

115 + ///

116 + /// Unknown argument keys are ignored, as in Transformers. Argument types and the

117 + /// `xml-inline` tag pattern are checked here rather than at parse time.

118 + pub fn new(scope: &str, kind: ContentKind, args: &Map<String, Value>) -> Result<Self> {

119 + let context = format!("{scope}: content_args");

120 + let args = Value::Object(args.clone());

121 + Ok(match kind {

122 + ContentKind::Text => Self::Text(deserialize(&args, &context)?),

123 + // Number conversion ignores `strip`, but a malformed value is still rejected.

124 + ContentKind::Int => {

125 + deserialize::<TextArgs>(&args, &context)?;

126 + Self::Int

127 + }

128 + ContentKind::Float => {

129 + deserialize::<TextArgs>(&args, &context)?;

130 + Self::Float

131 + }

132 + ContentKind::Bool => Self::Bool(deserialize(&args, &context)?),

133 + ContentKind::Json => {

134 + let args: JsonArgs = deserialize(&args, &context)?;

135 + if args

136 + .string_delims

137 + .iter()

138 + .any(|(open, close)| open.is_empty() || close.is_empty())

139 + {

140 + return Err(invalid!("{context}: string_delims cannot be empty strings"));

141 + }

142 + Self::Json(args)

143 + }

144 + ContentKind::XmlInline => {

145 + let spec: XmlInlineSpec = deserialize(&args, &context)?;

146 + let tag_pattern = compile_regex(scope, &spec.tag_pattern)?;

147 + let group_info = tag_pattern.group_info();

148 + let Some(key_group) = group_info.to_index(Default::default(), "key") else {

149 + return Err(invalid!(

150 + "{scope}: xml-inline: tag_pattern must have a named group 'key'. Pattern: {}",

151 + spec.tag_pattern

152 + ));

153 + };

154 + let value_group = group_info.to_index(Default::default(), "value");

155 + Self::XmlInline(XmlInlineArgs {

156 + tag_pattern,

157 + key_group,

158 + value_group,

159 + value_parser: ValueParserSpec::compile(scope, spec.value_parser)?,

160 + merge_duplicates: spec.merge_duplicates,

161 + })

162 + }

163 + ContentKind::KvLines => {

164 + let spec: KvLinesSpec = deserialize(&args, &context)?;

165 + if spec.line_sep.is_empty() {

166 + return Err(invalid!("{context}: line_sep cannot be empty"));

167 + }

168 + Self::KvLines(KvLinesArgs {

169 + text: spec.text,

170 + line_sep: spec.line_sep,

171 + kv_sep: spec.kv_sep,

172 + value_parser: ValueParserSpec::compile(scope, spec.value_parser)?,

173 + })

174 + }

175 + })

176 + }

177 +

178 + /// Parse one region body.

179 + pub fn parse(&self, text: &str) -> unified::Result<Value> {

180 + match self {

181 + Self::Text(args) => Ok(Value::String(args.apply(text).to_string())),

182 + // Numbers follow the tool-parameter conversion.

183 + Self::Int => convert_integer_text(text)

184 + .ok_or_else(|| parsing_failed!("int: invalid literal {text:?}")),

185 + Self::Float => convert_number_text(text)

186 + .ok_or_else(|| parsing_failed!("float: invalid or non-finite literal {text:?}")),

187 + Self::Bool(args) => {

188 + let text = args.apply(text).to_lowercase();

189 + Ok(Value::Bool(text == "true" || text == "1"))

190 + }

191 + Self::Json(args) => args.parse(text),

192 + Self::XmlInline(args) => args.parse(text),

193 + Self::KvLines(args) => args.parse(text),

194 + }

195 + }

196 +}

197 +

198 +impl ValueParserSpec {

199 + /// Build the nested parser, if any.

200 + fn compile(scope: &str, spec: Option<Self>) -> Result<Option<Box<ContentParser>>> {

201 + spec.map(|spec| {

202 + ContentParser::new(&format!("{scope}: value_parser"), spec.name, &spec.args)

203 + .map(Box::new)

204 + })

205 + .transpose()

206 + }

207 +}

208 +

209 +impl TextArgs {

210 + /// Apply `_text`: strip surrounding whitespace unless disabled.

211 + pub fn apply(self, text: &str) -> &str {

212 + if self.strip { text.trim() } else { text }

213 + }

214 +}

215 +

216 +impl JsonArgs {

217 + /// JSON parser with optional dialect knobs for LLM-emitted quirks.

218 + fn parse(&self, text: &str) -> unified::Result<Value> {

219 + if !self.string_delims.is_empty() && text.contains([LAX_OPEN, LAX_CLOSE]) {

220 + return Err(parsing_failed!(

221 + "json: input contains reserved sentinel characters (\\x01/\\x02); cannot parse safely."

222 + ));

223 + }

224 +

225 + let mut working = text.to_string();

226 + let mut captured: Vec<String> = Vec::new();

227 + for (open, close) in &self.string_delims {

228 + working = extract_delimited(&working, open, close, &mut captured);

229 + }

230 +

231 + if self.unquoted_keys {

232 + working = quote_bare_keys(&working);

233 + }

234 +

235 + for (index, string) in captured.iter().enumerate() {

236 + let quoted = serde_json::to_string(string).map_err(|error| {

237 + parsing_failed!("json: failed to quote string: {}", error.as_report())

238 + })?;

239 + working = working.replace(&format!("{LAX_OPEN}{index}{LAX_CLOSE}"), &quoted);

240 + }

241 +

242 + match serde_json::from_str(&working) {

243 + Ok(value) => Ok(value),

244 + Err(_) if self.allow_non_json => Ok(Value::String(self.text.apply(text).to_string())),

245 + Err(error) if working == text => Err(parsing_failed!(

246 + "json parser could not parse region as JSON.\nContent: {text:?}\nError: {}",

247 + error.as_report()

248 + )),

249 + Err(error) => Err(parsing_failed!(

250 + "json: could not parse after dialect transforms.\nOriginal: {text:?}\nTransformed: {working:?}\nError: {}",

251 + error.as_report()

252 + )),

253 + }

254 + }

255 +}

256 +

257 +impl XmlInlineArgs {

258 + /// Parse shallow XML-ish tags into a dict.

259 + fn parse(&self, text: &str) -> unified::Result<Value> {

260 + let mut out = Map::new();

261 + for captures in self.tag_pattern.captures_iter(text) {

262 + let key =

263 + captures.get_group(self.key_group).map(|span| &text[span.range()]).ok_or_else(

264 + || parsing_failed!("xml-inline: named group 'key' did not participate"),

265 + )?;

266 + let raw = match self.value_group {

267 + // The pattern has no `value` group.

268 + None => Some(""),

269 + Some(index) => captures.get_group(index).map(|span| &text[span.range()]),

270 + };

271 + let value = match raw {

272 + Some(raw) => sub_parse(raw, self.value_parser.as_deref())?,

273 + None if self.value_parser.is_none() => Value::Null,

274 + None => {

275 + return Err(parsing_failed!(

276 + "xml-inline: named group 'value' did not participate"

277 + ));

278 + }

279 + };

280 + match out.get_mut(key) {

281 + Some(existing) if self.merge_duplicates => match existing {

282 + Value::Array(values) => values.push(value),

283 + _ => {

284 + let first = existing.take();

285 + *existing = Value::Array(vec![first, value]);

286 + }

287 + },

288 + _ => {

289 + out.insert(key.to_string(), value);

290 + }

291 + }

292 + }

293 + Ok(Value::Object(out))

294 + }

295 +}

296 +

297 +impl KvLinesArgs {

298 + /// Parse line-delimited `key<sep>value` pairs into a dict.

299 + fn parse(&self, text: &str) -> unified::Result<Value> {

300 + let mut out = Map::new();

301 + for line in text.split(self.line_sep.as_str()) {

302 + let line = self.text.apply(line);

303 + let Some((key, value)) = line.split_once(self.kv_sep.as_str()) else {

304 + continue;

305 + };

306 + let (key, value) = (self.text.apply(key), self.text.apply(value));

307 + out.insert(

308 + key.to_string(),

309 + sub_parse(value, self.value_parser.as_deref())?,

310 + );

311 + }

312 + Ok(Value::Object(out))

313 + }

314 +}

315 +

316 +/// Parse `raw` with an optional nested value parser.

317 +fn sub_parse(raw: &str, value_parser: Option<&ContentParser>) -> unified::Result<Value> {

318 + match value_parser {

319 + None => Ok(Value::String(raw.to_string())),

320 + Some(parser) => parser.parse(raw),

321 + }

322 +}

323 +

324 +/// Replace every `open(.*?)close` span with an indexed sentinel, collecting the

325 +/// inner strings. Equivalent to the non-overlapping, left-to-right `re.sub` with

326 +/// `re.DOTALL` used by Transformers.

327 +fn extract_delimited(text: &str, open: &str, close: &str, captured: &mut Vec<String>) -> String {

328 + let mut out = String::with_capacity(text.len());

329 + let mut rest = text;

330 + while let Some(start) = rest.find(open) {

331 + let inner_start = start + open.len();

332 + // No close after this open means no close after any later open either.

333 + let Some(inner_len) = rest[inner_start..].find(close) else {

334 + break;

335 + };

336 + out.push_str(&rest[..start]);

337 + captured.push(rest[inner_start..inner_start + inner_len].to_string());

338 + out.push(LAX_OPEN);

339 + out.push_str(&(captured.len() - 1).to_string());

340 + out.push(LAX_CLOSE);

341 + rest = &rest[inner_start + inner_len + close.len()..];

342 + }

343 + out.push_str(rest);

344 + out

345 +}

346 +

347 +/// Quote bare-identifier keys: Transformers' `(?<=[{,])(\w+):` → `"\1":`.

348 +///

349 +/// The Rust regex engine has no lookbehind, so the preceding `{` or `,` is

350 +/// matched and written back. Matches cannot overlap either way, so the rewrite is

351 +/// equivalent.

352 +fn quote_bare_keys(text: &str) -> String {

353 + static BARE_KEY: std::sync::LazyLock<Regex> =

354 + std::sync::LazyLock::new(|| Regex::new(r"([{,])(\w+):").expect("valid regex"));

355 +

356 + let mut out = String::with_capacity(text.len());

357 + let mut last = 0;

358 + for captures in BARE_KEY.captures_iter(text) {

359 + let whole = captures.get_match().expect("match");

360 + let separator = captures.get_group(1).expect("group 1");

361 + let key = captures.get_group(2).expect("group 2");

362 + out.push_str(&text[last..whole.start()]);

363 + out.push_str(&text[separator.range()]);

364 + out.push('"');

365 + out.push_str(&text[key.range()]);

366 + out.push_str("\":");

367 + last = whole.end();

368 + }

369 + out.push_str(&text[last..]);

370 + out

371 +}

372 +

373 +/// Compile a Python-syntax regex with `re.DOTALL`, as Transformers does.

374 +pub(super) fn compile_regex(scope: &str, pattern: &str) -> Result<Regex> {

375 + Regex::builder()

376 + .syntax(syntax::Config::new().dot_matches_new_line(true))

377 + .build(pattern)

378 + .map_err(|error| {

379 + unsupported!(

380 + "{scope}: regex {pattern:?} is not supported by the Rust regex engine: {}",

381 + error.as_report()

382 + )

383 + })

384 +}

385 +

386 +fn default_strip() -> bool {

387 + true

388 +}

389 +

390 +fn default_line_sep() -> String {

391 + "\n".to_string()

392 +}

393 +

394 +fn default_kv_sep() -> String {

395 + ":".to_string()

396 +}

397 +

398 +#[cfg(test)]

399 +mod tests {

400 + use serde_json::json;

401 +

402 + use super::*;

403 +

404 + fn kind(name: &str) -> ContentKind {

405 + serde_json::from_value(json!(name)).unwrap()

406 + }

407 +

408 + fn parser(name: &str, args: Value) -> ContentParser {

409 + ContentParser::new("test", kind(name), args.as_object().unwrap()).unwrap()

410 + }

411 +

412 + #[test]

413 + fn scalar_parsers_convert_like_tool_parameters() {

414 + let int = parser("int", json!({}));

415 + assert_eq!(int.parse(" 42\n").unwrap(), json!(42));

416 + assert_eq!(int.parse("-1_000").unwrap(), json!(-1000));

417 + assert!(int.parse("1__0").is_err());

418 + assert!(int.parse("4.2").is_err());

419 +

420 + let float = parser("float", json!({}));

421 + assert_eq!(float.parse("1.5").unwrap(), json!(1.5));

422 + assert_eq!(float.parse(" 1e3 ").unwrap(), json!(1000.0));

423 + assert!(float.parse("inf").is_err(), "JSON cannot represent inf");

424 +

425 + let boolean = parser("bool", json!({}));

426 + assert_eq!(boolean.parse(" TRUE ").unwrap(), json!(true));

427 + assert_eq!(boolean.parse("1").unwrap(), json!(true));

428 + assert_eq!(boolean.parse("yes").unwrap(), json!(false));

429 +

430 + let text = parser("text", json!({"strip": false}));

431 + assert_eq!(text.parse(" a \n").unwrap(), json!(" a \n"));

432 + assert_eq!(

433 + parser("text", json!({})).parse(" a \n").unwrap(),

434 + json!("a")

435 + );

436 + }

437 +

438 + #[test]

439 + fn json_dialects_match_gemma4_syntax() {

440 + let json = parser(

441 + "json",

442 + json!({"unquoted_keys": true, "string_delims": [["<|\"|>", "<|\"|>"]]}),

443 + );

444 + let value = json

445 + .parse(

446 + r#"{bool_value:true,list_value:[<|"|>foo<|"|>,<|"|>bar<|"|>],null_value:null,number_value:1,string_value:<|"|>a, b: {c}<|"|>,struct_value:{foo:<|"|>bar<|"|>}}"#,

447 + )

448 + .unwrap();

449 + assert_eq!(

450 + value,

451 + json!({

452 + "bool_value": true,

453 + "list_value": ["foo", "bar"],

454 + "null_value": null,

455 + "number_value": 1,

456 + "string_value": "a, b: {c}",

457 + "struct_value": {"foo": "bar"},

458 + })

459 + );

460 + assert!(json.parse("{a:\u{1}}").is_err());

461 + }

462 +

463 + #[test]

464 + fn json_allow_non_json_returns_stripped_text() {

465 + let json = parser("json", json!({"allow_non_json": true}));

466 + assert_eq!(json.parse(" celsius \n").unwrap(), json!("celsius"));

467 + assert_eq!(json.parse(" [1, 2] ").unwrap(), json!([1, 2]));

468 + assert!(parser("json", json!({})).parse("celsius").is_err());

469 + }

470 +

471 + #[test]

472 + fn xml_inline_parses_tags_with_value_parser_and_merging() {

473 + let xml = parser(

474 + "xml-inline",

475 + json!({

476 + "tag_pattern": r"<parameter=(?P<key>\w+)>\s*(?P<value>.*?)\s*</parameter>",

477 + "value_parser": {"name": "json", "args": {"allow_non_json": true}},

478 + "merge_duplicates": true,

479 + }),

480 + );

481 + let value = xml

482 + .parse(

483 + "<parameter=locations>\n[{\"city\": \"Paris\"}]\n</parameter>\n\

484 + <parameter=unit>\ncelsius\n</parameter><parameter=unit>2</parameter>",

485 + )

486 + .unwrap();

487 + assert_eq!(

488 + value,

489 + json!({"locations": [{"city": "Paris"}], "unit": ["celsius", 2]})

490 + );

491 + }

492 +

493 + #[test]

494 + fn kv_lines_split_and_strip() {

495 + let kv = parser("kv-lines", json!({"value_parser": {"name": "int"}}));

496 + assert_eq!(

497 + kv.parse(" a : 1 \n\nnot a pair\nb:2").unwrap(),

498 + json!({"a": 1, "b": 2})

499 + );

500 + }

501 +

502 + #[test]

503 + fn invalid_arguments_are_rejected_at_load() {

504 + let error = |name: &str, args: Value| {

505 + ContentParser::new("Field 'x'", kind(name), args.as_object().unwrap()).unwrap_err()

506 + };

507 + expect_test::expect![[r#"

508 + [

509 + Invalid {

510 + message: "Field 'x': content_args.strip: invalid type: string \"no\", expected a boolean",

511 + },

512 + Invalid {

513 + message: "Field 'x': content_args.string_delims[0]: invalid length 1, expected a tuple of size 2",

514 + },

515 + Invalid {

516 + message: "Field 'x': content_args: missing field `tag_pattern`",

517 + },

518 + Invalid {

519 + message: "Field 'x': xml-inline: tag_pattern must have a named group 'key'. Pattern: <(?P<k>\\w+)>",

520 + },

521 + Invalid {

522 + message: "Field 'x': content_args.value_parser.name: unknown variant `yaml`, expected one of `text`, `int`, `float`, `bool`, `json`, `xml-inline`, `kv-lines`",

523 + },

524 + ]

525 + "#]].assert_debug_eq(&[

526 + error("text", json!({"strip": "no"})),

527 + error("json", json!({"string_delims": [["<s>"]]})),

528 + error("xml-inline", json!({})),

529 + error("xml-inline", json!({"tag_pattern": r"<(?P<k>\w+)>"})),

530 + error("kv-lines", json!({"value_parser": {"name": "yaml"}})),

531 + ]);

532 + assert!(matches!(

533 + error("xml-inline", json!({"tag_pattern": r"(?<=a)b"})),

534 + super::super::HfTemplateError::Unsupported { .. }

535 + ));

536 + }

537 +}

rust/src/parser/src/unified/hf/fixtures/differential.json new

+440

@@ -0,0 +1,440 @@

1 +{

2 + "templates": {

3 + "gemma4": {

4 + "defaults": {

5 + "role": "assistant"

6 + },

7 + "start_anchor": [

8 + "<|turn>model\n",

9 + "<tool_response|>"

10 + ],

11 + "fields": {

12 + "thinking": {

13 + "open": "<|channel>thought\n",

14 + "close": "<channel|>",

15 + "content": "text"

16 + },

17 + "tool_calls": {

18 + "open_pattern": "<\\|tool_call>call:(?P<name>\\w+)",

19 + "close": "<tool_call|>",

20 + "repeats": true,

21 + "content": "json",

22 + "content_args": {

23 + "unquoted_keys": true,

24 + "string_delims": [

25 + [

26 + "<|\"|>",

27 + "<|\"|>"

28 + ]

29 + ]

30 + },

31 + "transform": {

32 + "type": "function",

33 + "function": {

34 + "name": "{name}",

35 + "arguments": "{content}"

36 + }

37 + }

38 + },

39 + "content": {

40 + "close": [

41 + "<turn|>",

42 + "<|tool_response>",

43 + "<eos>"

44 + ],

45 + "content": "text"

46 + }

47 + }

48 + },

49 + "invoke": {

50 + "defaults": {

51 + "role": "assistant"

52 + },

53 + "start_anchor": "<|begin|>assistant",

54 + "fields": {

55 + "reasoning_content": {

56 + "open_pattern": "to=self<\\|msg\\|>",

57 + "close": "<|pause|>",

58 + "content": "text"

59 + },

60 + "tool_calls": {

61 + "open_pattern": "<invoke\\b[^>]*?\\bname=\"(?P<name>[^\"]+)\">",

62 + "close": "</invoke>",

63 + "repeats": true,

64 + "content": "xml-inline",

65 + "content_args": {

66 + "tag_pattern": "<param\\b[^>]*?\\bname=\"(?P<key>[^\"]+)\"[^>]*?>(?P<value>.*?)</param>",

67 + "value_parser": {

68 + "name": "json",

69 + "args": {

70 + "allow_non_json": true

71 + }

72 + }

73 + },

74 + "transform": {

75 + "type": "function",

76 + "function": {

77 + "name": "{name}",

78 + "arguments": "{content}"

79 + }

80 + }

81 + },

82 + "content": {

83 + "open_pattern": "to=(?:user|note)<\\|msg\\|>",

84 + "close": [

85 + "<|end|>",

86 + "<|pause|>"

87 + ],

88 + "repeats": true,

89 + "join": "",

90 + "content": "text"

91 + }

92 + }

93 + },

94 + "kv_tools": {

95 + "start_anchor": "<|im_start|>assistant\n",

96 + "fields": {

97 + "thinking": {

98 + "open": "<think>",

99 + "close": "</think>"

100 + },

101 + "tool_calls": {

102 + "open_pattern": "<tool_call>\\s*<function=(?P<name>\\w+)>\\n",

103 + "close": "</tool_call>",

104 + "repeats": true,

105 + "content": "kv-lines",

106 + "transform": {

107 + "type": "function",

108 + "function": {

109 + "name": "{name}",

110 + "arguments": "{content}"

111 + }

112 + }

113 + },

114 + "content": {

115 + "close": "<|im_end|>"

116 + }

117 + }

118 + }

119 + },

120 + "cases": [

121 + {

122 + "template": "gemma4",

123 + "prefix": "",

124 + "text": "Hello there.",

125 + "tools": null,

126 + "expected": {

127 + "content": "Hello there."

128 + }

129 + },

130 + {

131 + "template": "gemma4",

132 + "prefix": "",

133 + "text": " \n Hello\n\n",

134 + "tools": null,

135 + "expected": {

136 + "content": "Hello"

137 + }

138 + },

139 + {

140 + "template": "gemma4",

141 + "prefix": "",

142 + "text": "<|channel>thought\n plan\n<channel|>\n\nAnswer: 天气很好 ",

143 + "tools": null,

144 + "expected": {

145 + "reasoning": "plan",

146 + "content": "Answer: 天气很好"

147 + }

148 + },

149 + {

150 + "template": "gemma4",

151 + "prefix": "",

152 + "text": "<|channel>thought\nunterminated reasoning",

153 + "tools": null,

154 + "expected": {

155 + "reasoning": "unterminated reasoning"

156 + }

157 + },

158 + {

159 + "template": "gemma4",

160 + "prefix": "",

161 + "text": "Before <|tool_call>call:f{x:<|\"|>a, b<|\"|>,y:[1,2.5,-3],z:{w:true}}<tool_call|> after",

162 + "tools": null,

163 + "expected": {

164 + "content": "after",

165 + "tool_calls": [

166 + {

167 + "name": "f",

168 + "arguments": {

169 + "x": "a, b",

170 + "y": [

171 + 1,

172 + 2.5,

173 + -3

174 + ],

175 + "z": {

176 + "w": true

177 + }

178 + }

179 + }

180 + ]

181 + }

182 + },

183 + {

184 + "template": "gemma4",

185 + "prefix": "",

186 + "text": "<|tool_call>call:f{}<tool_call|><|tool_call>call:g{n:1}<tool_call|>",

187 + "tools": null,

188 + "expected": {

189 + "tool_calls": [

190 + {

191 + "name": "f",

192 + "arguments": {}

193 + },

194 + {

195 + "name": "g",

196 + "arguments": {

197 + "n": 1

198 + }

199 + }

200 + ]

201 + }

202 + },

203 + {

204 + "template": "gemma4",

205 + "prefix": "",

206 + "text": "<|tool_call>call:f{a:<|\"|>unterminated",

207 + "tools": null,

208 + "expected": {

209 + "error": "ValueError"

210 + }

211 + },

212 + {

213 + "template": "gemma4",

214 + "prefix": "",

215 + "text": "text <|tool_call> not a call",

216 + "tools": null,

217 + "expected": {

218 + "content": "text <|tool_call> not a call"

219 + }

220 + },

221 + {

222 + "template": "gemma4",

223 + "prefix": "",

224 + "text": "<|tool_call>call:f{a:1}",

225 + "tools": null,

226 + "expected": {

227 + "tool_calls": [

228 + {

229 + "name": "f",

230 + "arguments": {

231 + "a": 1

232 + }

233 + }

234 + ]

235 + }

236 + },

237 + {

238 + "template": "gemma4",

239 + "prefix": "<|turn>user\nhi<turn|>\n<|turn>model\n",

240 + "text": "Hi!",

241 + "tools": null,

242 + "expected": {

243 + "content": "Hi!"

244 + }

245 + },

246 + {

247 + "template": "gemma4",

248 + "prefix": "<|turn>model\n<|tool_call>call:f{}<tool_call|><|tool_response>response:f{}<tool_response|><|channel>thought\n",

249 + "text": "Done.<channel|>It is sunny.",

250 + "tools": null,

251 + "expected": {

252 + "reasoning": "Done.",

253 + "content": "It is sunny."

254 + }

255 + },

256 + {

257 + "template": "gemma4",

258 + "prefix": "<|turn>model\n<|channel>thought\n<channel|>",

259 + "text": "Plain answer",

260 + "tools": null,

261 + "expected": {

262 + "content": "Plain answer"

263 + }

264 + },

265 + {

266 + "template": "invoke",

267 + "prefix": "",

268 + "text": "to=self<|msg|>think<|pause|>to=user<|msg|>A<|pause|>to=note<|msg|> B <|end|>",

269 + "tools": null,

270 + "expected": {

271 + "reasoning": "think",

272 + "content": "AB"

273 + }

274 + },

275 + {

276 + "template": "invoke",

277 + "prefix": "",

278 + "text": "noise <invoke name=\"get\" id=\"1\"><param name=\"city\">\"Paris\"</param><param name=\"n\">3</param><param name=\"raw\">not json</param></invoke>",

279 + "tools": null,

280 + "expected": {}

281 + },

282 + {

283 + "template": "invoke",

284 + "prefix": "",

285 + "text": "<invoke name=\"get\"><param name=\"k\">1</param><param name=\"k\">2</param></invoke>",

286 + "tools": null,

287 + "expected": {

288 + "tool_calls": [

289 + {

290 + "name": "get",

291 + "arguments": {

292 + "k": 2

293 + }

294 + }

295 + ]

296 + }

297 + },

298 + {

299 + "template": "invoke",

300 + "prefix": "",

301 + "text": "bare text only",

302 + "tools": null,

303 + "expected": {}

304 + },

305 + {

306 + "template": "invoke",

307 + "prefix": "",

308 + "text": "to=user<|msg|>unterminated content",

309 + "tools": null,

310 + "expected": {

311 + "content": "unterminated content"

312 + }

313 + },

314 + {

315 + "template": "invoke",

317 + "text": "to=user<|msg|>hi<|end|>",

318 + "tools": null,

319 + "expected": {

320 + "content": "hi"

321 + }

322 + },

323 + {

324 + "template": "kv_tools",

325 + "prefix": "",

326 + "text": "<tool_call>\n<function=set_alarm>\nhour: 7\nratio: 0.5\nenabled: 1\nlabel: 7\nmaybe: None\n</tool_call>",

327 + "tools": [

328 + {

329 + "type": "function",

330 + "function": {

331 + "name": "set_alarm",

332 + "parameters": {

333 + "type": "object",

334 + "properties": {

335 + "hour": {

336 + "type": "integer"

337 + },

338 + "ratio": {

339 + "type": "number"

340 + },

341 + "enabled": {

342 + "type": "boolean"

343 + },

344 + "label": {

345 + "type": "string"

346 + },

347 + "maybe": {

348 + "anyOf": [

349 + {

350 + "type": "integer"

351 + },

352 + {

353 + "type": "null"

354 + }

355 + ]

356 + }

357 + }

358 + }

359 + }

360 + }

361 + ],

362 + "expected": {

363 + "tool_calls": [

364 + {

365 + "name": "set_alarm",

366 + "arguments": {

367 + "hour": 7,

368 + "ratio": 0.5,

369 + "enabled": true,

370 + "label": "7",

371 + "maybe": null

372 + }

373 + }

374 + ]

375 + }

376 + },

377 + {

378 + "template": "kv_tools",

379 + "prefix": "",

380 + "text": "<tool_call>\n<function=set_alarm>\nhour: seven\nratio: 1e3\nenabled: nope\n</tool_call>",

381 + "tools": [

382 + {

383 + "type": "function",

384 + "function": {

385 + "name": "set_alarm",

386 + "parameters": {

387 + "type": "object",

388 + "properties": {

389 + "hour": {

390 + "type": "integer"

391 + },

392 + "ratio": {

393 + "type": "number"

394 + },

395 + "enabled": {

396 + "type": "boolean"

397 + },

398 + "label": {

399 + "type": "string"

400 + },

401 + "maybe": {

402 + "anyOf": [

403 + {

404 + "type": "integer"

405 + },

406 + {

407 + "type": "null"

408 + }

409 + ]

410 + }

411 + }

412 + }

413 + }

414 + }

415 + ],

416 + "expected": {

417 + "tool_calls": [

418 + {

419 + "name": "set_alarm",

420 + "arguments": {

421 + "hour": "seven",

422 + "ratio": 1000,

423 + "enabled": "nope"

424 + }

425 + }

426 + ]

427 + }

428 + },

429 + {

430 + "template": "kv_tools",

431 + "prefix": "<|im_start|>assistant\n<think>\n",

432 + "text": "\n reasoning \n</think>\n\nanswer<|im_end|>",

433 + "tools": null,

434 + "expected": {

435 + "reasoning": "reasoning",

436 + "content": "answer"

437 + }

438 + }

439 + ]

440 +}

rust/src/parser/src/unified/hf/fixtures/generate_differential.py new

+264

@@ -0,0 +1,264 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Regenerate `differential.json` from Transformers `parse_response`.

4 +

5 +Requires Transformers with `transformers.utils.chat_parsing` (>= 5.13); the

6 +expected values in the checked-in file were produced at commit 6d43ab4008.

7 +

8 + python generate_differential.py > differential.json

9 +"""

10 +

11 +import json

12 +

13 +from transformers.utils.chat_parsing import parse_response

14 +

15 +GEMMA4 = {

16 + "defaults": {"role": "assistant"},

17 + "start_anchor": ["<|turn>model\n", "<tool_response|>"],

18 + "fields": {

19 + "thinking": {

20 + "open": "<|channel>thought\n",

21 + "close": "<channel|>",

22 + "content": "text",

23 + },

24 + "tool_calls": {

25 + "open_pattern": r"<\|tool_call>call:(?P<name>\w+)",

26 + "close": "<tool_call|>",

27 + "repeats": True,

28 + "content": "json",

29 + "content_args": {

30 + "unquoted_keys": True,

31 + "string_delims": [['<|"|>', '<|"|>']],

32 + },

33 + "transform": {

34 + "type": "function",

35 + "function": {"name": "{name}", "arguments": "{content}"},

36 + },

37 + },

38 + "content": {

39 + "close": ["<turn|>", "<|tool_response>", "<eos>"],

40 + "content": "text",

41 + },

42 + },

43 +}

44 +

45 +INVOKE = {

46 + "defaults": {"role": "assistant"},

47 + "start_anchor": "<|begin|>assistant",

48 + "fields": {

49 + "reasoning_content": {

50 + "open_pattern": r"to=self<\|msg\|>",

51 + "close": "<|pause|>",

52 + "content": "text",

53 + },

54 + "tool_calls": {

55 + "open_pattern": r'<invoke\b[^>]*?\bname="(?P<name>[^"]+)">',

56 + "close": "</invoke>",

57 + "repeats": True,

58 + "content": "xml-inline",

59 + "content_args": {

60 + "tag_pattern": (

61 + r'<param\b[^>]*?\bname="(?P<key>[^"]+)"[^>]*?>'

62 + r"(?P<value>.*?)</param>"

63 + ),

64 + "value_parser": {"name": "json", "args": {"allow_non_json": True}},

65 + },

66 + "transform": {

67 + "type": "function",

68 + "function": {"name": "{name}", "arguments": "{content}"},

69 + },

70 + },

71 + "content": {

72 + "open_pattern": r"to=(?:user|note)<\|msg\|>",

73 + "close": ["<|end|>", "<|pause|>"],

74 + "repeats": True,

75 + "join": "",

76 + "content": "text",

77 + },

78 + },

79 +}

80 +

81 +KV_TOOLS = {

82 + "start_anchor": "<|im_start|>assistant\n",

83 + "fields": {

84 + "thinking": {"open": "<think>", "close": "</think>"},

85 + "tool_calls": {

86 + "open_pattern": r"<tool_call>\s*<function=(?P<name>\w+)>\n",

87 + "close": "</tool_call>",

88 + "repeats": True,

89 + "content": "kv-lines",

90 + "transform": {

91 + "type": "function",

92 + "function": {"name": "{name}", "arguments": "{content}"},

93 + },

94 + },

95 + "content": {"close": "<|im_end|>"},

96 + },

97 +}

98 +

99 +ALARM_TOOLS = [

100 + {

101 + "type": "function",

102 + "function": {

103 + "name": "set_alarm",

104 + "parameters": {

105 + "type": "object",

106 + "properties": {

107 + "hour": {"type": "integer"},

108 + "ratio": {"type": "number"},

109 + "enabled": {"type": "boolean"},

110 + "label": {"type": "string"},

111 + "maybe": {"anyOf": [{"type": "integer"}, {"type": "null"}]},

112 + },

113 + },

114 + },

115 + }

116 +]

117 +

118 +CASES = [

119 + ("gemma4", GEMMA4, "", "Hello there.", None),

120 + ("gemma4", GEMMA4, "", " \n Hello\n\n", None),

121 + (

122 + "gemma4",

123 + GEMMA4,

124 + "",

125 + "<|channel>thought\n plan\n<channel|>\n\nAnswer: 天气很好 ",

126 + None,

127 + ),

128 + ("gemma4", GEMMA4, "", "<|channel>thought\nunterminated reasoning", None),

129 + (

130 + "gemma4",

131 + GEMMA4,

132 + "",

133 + 'Before <|tool_call>call:f{x:<|"|>a, b<|"|>,y:[1,2.5,-3],z:{w:true}}'

134 + "<tool_call|> after",

135 + None,

136 + ),

137 + (

138 + "gemma4",

139 + GEMMA4,

140 + "",

141 + "<|tool_call>call:f{}<tool_call|><|tool_call>call:g{n:1}<tool_call|>",

142 + None,

143 + ),

144 + ("gemma4", GEMMA4, "", '<|tool_call>call:f{a:<|"|>unterminated', None),

145 + ("gemma4", GEMMA4, "", "text <|tool_call> not a call", None),

146 + ("gemma4", GEMMA4, "", "<|tool_call>call:f{a:1}", None),

147 + ("gemma4", GEMMA4, "<|turn>user\nhi<turn|>\n<|turn>model\n", "Hi!", None),

148 + (

149 + "gemma4",

150 + GEMMA4,

151 + "<|turn>model\n<|tool_call>call:f{}<tool_call|><|tool_response>response:f{}<tool_response|><|channel>thought\n",

152 + "Done.<channel|>It is sunny.",

153 + None,

154 + ),

155 + (

156 + "gemma4",

157 + GEMMA4,

158 + "<|turn>model\n<|channel>thought\n<channel|>",

159 + "Plain answer",

160 + None,

161 + ),

162 + (

163 + "invoke",

164 + INVOKE,

165 + "",

166 + "to=self<|msg|>think<|pause|>to=user<|msg|>A<|pause|>to=note<|msg|> B <|end|>",

167 + None,

168 + ),

169 + (

170 + "invoke",

171 + INVOKE,

172 + "",

173 + 'noise <invoke name="get" id="1"><param name="city">"Paris"</param>'

174 + '<param name="n">3</param><param name="raw">not json</param></invoke>',

175 + None,

176 + ),

177 + (

178 + "invoke",

179 + INVOKE,

180 + "",

181 + '<invoke name="get"><param name="k">1</param>'

182 + '<param name="k">2</param></invoke>',

183 + None,

184 + ),

185 + ("invoke", INVOKE, "", "bare text only", None),

186 + ("invoke", INVOKE, "", "to=user<|msg|>unterminated content", None),

187 + (

188 + "invoke",

189 + INVOKE,

190 + "<|begin|>user<|end|><|begin|>assistant",

191 + "to=user<|msg|>hi<|end|>",

192 + None,

193 + ),

194 + (

195 + "kv_tools",

196 + KV_TOOLS,

197 + "",

198 + "<tool_call>\n<function=set_alarm>\nhour: 7\nratio: 0.5\nenabled: 1\n"

199 + "label: 7\nmaybe: None\n</tool_call>",

200 + ALARM_TOOLS,

201 + ),

202 + (

203 + "kv_tools",

204 + KV_TOOLS,

205 + "",

206 + "<tool_call>\n<function=set_alarm>\nhour: seven\nratio: 1e3\n"

207 + "enabled: nope\n</tool_call>",

208 + ALARM_TOOLS,

209 + ),

210 + (

211 + "kv_tools",

212 + KV_TOOLS,

213 + "<|im_start|>assistant\n<think>\n",

214 + "\n reasoning \n</think>\n\nanswer<|im_end|>",

215 + None,

216 + ),

217 +]

218 +

219 +

220 +def normalize(message):

221 + out = {}

222 + reasoning = message.get("thinking", message.get("reasoning_content"))

223 + if reasoning:

224 + out["reasoning"] = reasoning

225 + if message.get("content"):

226 + out["content"] = message["content"]

227 + calls = [

228 + {"name": call["function"]["name"], "arguments": call["function"]["arguments"]}

229 + for call in message.get("tool_calls", [])

230 + ]

231 + if calls:

232 + out["tool_calls"] = calls

233 + return out

234 +

235 +

236 +def main():

237 + templates = {}

238 + cases = []

239 + for template_name, template, prefix, text, tools in CASES:

240 + templates[template_name] = template

241 + try:

242 + expected = normalize(

243 + parse_response(text, template, prefix=prefix, tools=tools)

244 + )

245 + except Exception as error: # noqa: BLE001

246 + expected = {"error": type(error).__name__}

247 + cases.append(

248 + {

249 + "template": template_name,

250 + "prefix": prefix,

251 + "text": text,

252 + "tools": tools,

253 + "expected": expected,

254 + }

255 + )

256 + print(

257 + json.dumps(

258 + {"templates": templates, "cases": cases}, indent=2, ensure_ascii=False

259 + )

260 + )

261 +

262 +

263 +if __name__ == "__main__":

264 + main()

rust/src/parser/src/unified/hf/mod.rs new

+53

@@ -0,0 +1,53 @@

1 +// SPDX-License-Identifier: Apache-2.0

2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +//! Unified parser driven by a checkpoint's Hugging Face `response_template`.

5 +//!

6 +//! The renderer executes a checkpoint's `chat_template`; this parser executes its

7 +//! `response_template`, the declarative output-protocol description introduced by

8 +//! Transformers `chat_parsing` (`ResponseParser`).

9 +//!

10 +//! The template language and its value semantics (content parsers, transforms,

11 +//! tool-schema coercion) follow Transformers at

12 +//! <https://github.com/huggingface/transformers/tree/6d43ab4008/src/transformers/utils/chat_parsing>.

13 +//! The streaming executor is native: it emits ordered [`UnifiedParserEvent`]s

14 +//! instead of Transformers' region events and aggregated message dict.

15 +//!

16 +//! [`UnifiedParserEvent`]: super::UnifiedParserEvent

17 +

18 +mod content;

19 +mod parser;

20 +mod pattern;

21 +mod spec;

22 +mod template;

23 +mod transform;

24 +

25 +#[cfg(test)]

26 +mod tests;

27 +

28 +pub use parser::HfUnifiedParser;

29 +pub use template::ResponseTemplate;

30 +use thiserror::Error;

31 +use thiserror_ext::Macro;

32 +

33 +/// Result alias for response-template operations.

34 +pub type Result<T> = std::result::Result<T, HfTemplateError>;

35 +

36 +/// Errors produced while loading a `response_template`.

37 +///

38 +/// Failures while parsing generated output are

39 +/// [`UnifiedParserError::ParsingFailed`](super::UnifiedParserError::ParsingFailed), as for

40 +/// the other unified parsers.

41 +#[derive(Debug, Clone, Error, Macro)]

42 +#[thiserror_ext(macro(path = "crate::unified::hf", mangle))]

43 +pub enum HfTemplateError {

44 + /// The template violates the Transformers `response_template` schema.

45 + #[error("invalid response_template: {message}")]

46 + Invalid { message: String },

47 + /// The template is valid but uses a feature this implementation does not support.

48 + #[error("unsupported response_template: {message}")]

49 + Unsupported { message: String },

50 + /// The model provides no template.

51 + #[error("the model's tokenizer_config.json provides no response_template")]

52 + Missing,

53 +}

rust/src/parser/src/unified/hf/parser.rs new

+730

@@ -0,0 +1,730 @@

1 +// SPDX-License-Identifier: Apache-2.0

2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +//! Streaming executor for a compiled [`ResponseTemplate`].

5 +//!

6 +//! Follows `ResponseParser` in `response_parser.py`: a flat region state

7 +//! machine that, outside explicit regions, watches every explicit open plus the

8 +//! implicit region's close, and inside an explicit region watches only that

9 +//! region's close. Instead of region events and an aggregated dict, it emits

10 +//! ordered unified parser events:

11 +//!

12 +//! - text and reasoning stream as they arrive, with `strip` applied so that the

13 +//! concatenated deltas equal Transformers' stripped value. For a field without

14 +//! `repeats`, stripping spans all its occurrences: an interrupted field (text,

15 +//! tool call, more text) streams every occurrence, where Transformers keeps only

16 +//! the last;

17 +//! - a tool call starts at its opener when the opener captures the function

18 +//! name, and its arguments are emitted when the region closes.

19 +

20 +use std::mem::take;

21 +use std::sync::Arc;

22 +

23 +use serde::Deserialize;

24 +use serde_json::{Map, Value};

25 +use thiserror_ext::AsReport as _;

26 +use vllm_tokenizer::{DecodedText, DynTokenizer, TokenAnchor, TokenAttribution};

27 +use winnow::error::{ErrMode, ModalResult};

28 +use winnow::stream::{Partial, Stream};

29 +

30 +use super::pattern::{Captures, Resolution};

31 +use super::template::{

32 + FieldName, Region, RegionKind, Repeat, ResponseTemplate, TextRegion, TextRole, ToolCallRegion,

33 + Watch, WatchKind, WatchSet,

34 +};

35 +use crate::tool::{Tool, ToolCallDelta, ToolSchemas};

36 +use crate::unified::{

37 + Result, UnifiedParser, UnifiedParserError, UnifiedParserOutput, parsing_failed,

38 +};

39 +use crate::utils::{incomplete, parse_buffered_event, safe_text_len_mul};

40 +

41 +/// Prompt tail windows, in tokens, searched for the start anchor before

42 +/// decoding the whole prompt.

43 +const PROMPT_WINDOWS: [usize; 3] = [64, 512, 4096];

44 +

45 +/// Unified parser executing a checkpoint's `response_template`.

46 +///

47 +/// Original Python implementation:

48 +/// <https://github.com/huggingface/transformers/blob/6d43ab4008/src/transformers/utils/chat_parsing/response_parser.py>

49 +pub struct HfUnifiedParser {

50 + template: Arc<ResponseTemplate>,

51 + tool_schemas: ToolSchemas,

52 + tokenizer: DynTokenizer,

53 + buffer: DecodedText,

54 + mode: HfMode,

55 + /// Whether each region has closed an occurrence.

56 + closed: Vec<bool>,

57 + /// Strip state per region: per occurrence with `repeats`, per stream otherwise.

58 + strip: Vec<StripState>,

59 + emitted_tool_count: usize,

60 +}

61 +

62 +/// Parser mode: the region currently receiving text.

63 +enum HfMode {

64 + /// No implicit field is declared: text outside regions is discarded.

65 + Discard,

66 + /// The implicit region, opened lazily on its first byte.

67 + Implicit(Occurrence),

68 + /// An explicit region, opened by its open boundary.

69 + Explicit(Occurrence),

70 +}

71 +

72 +/// One occurrence of a region.

73 +struct Occurrence {

74 + region: usize,

75 + /// Whether the occurrence has content to close (`_opened`).

76 + opened: bool,

77 + captures: Captures,

78 + /// Opener text, returned by [`UnifiedParser::reset`].

79 + raw_open: String,

80 + /// Buffered body of a tool-call region.

81 + body: String,

82 + /// Index of the tool call started at the opener, if any.

83 + active_tool_index: Option<usize>,

84 + /// `join` separator still to emit before this occurrence's content.

85 + separator: Option<String>,

86 +}

87 +

88 +/// Strip-compatible streaming state of a text or reasoning occurrence.

89 +#[derive(Default)]

90 +struct StripState {

91 + enabled: bool,

92 + /// Whether a non-whitespace character has been seen.

93 + started: bool,

94 + /// Trailing whitespace held until more text arrives or the region closes.

95 + held: DecodedText,

96 +}

97 +

98 +/// One parsed event of the buffered input.

99 +enum HfEvent {

100 + /// Text routed into the current region.

101 + Text,

102 + /// A boundary of the current mode's watch list.

103 + Boundary { watch: Watch, captures: Captures },

104 +}

105 +

106 +/// A committable boundary match at the current position.

107 +struct BoundaryMatch {

108 + watch: Watch,

109 + len: usize,

110 + captures: Captures,

111 +}

112 +

113 +impl BoundaryMatch {

114 + /// Preference order of `_scan`: longest first, opens before closes, then by

115 + /// field name.

116 + fn key(&self, template: &ResponseTemplate) -> (std::cmp::Reverse<usize>, bool, FieldName) {

117 + (

118 + std::cmp::Reverse(self.len),

119 + self.watch.kind == WatchKind::Close,

120 + template.regions[self.watch.region].name,

121 + )

122 + }

123 +}

124 +

125 +/// How the event loop treats its input.

126 +#[derive(Clone, Copy)]

127 +enum Phase {

128 + /// Prompt text after the start anchor: state transitions only, no events.

129 + Prefill,

130 + /// Generated text; more may follow.

131 + Stream,

132 + /// Generated text at the end of the stream.

133 + Eof,

134 +}

135 +

136 +impl Phase {

137 + fn emits(self) -> bool {

138 + !matches!(self, Self::Prefill)

139 + }

140 +}

141 +

142 +impl HfUnifiedParser {

143 + /// Create a parser for one request.

144 + pub fn new(template: Arc<ResponseTemplate>, tools: &[Tool], tokenizer: DynTokenizer) -> Self {

145 + let mut parser = Self {

146 + closed: vec![false; template.regions.len()],

147 + strip: template.regions.iter().map(StripState::new).collect(),

148 + mode: HfMode::Discard,

149 + template,

150 + tool_schemas: ToolSchemas::from_tools(tools),

151 + tokenizer,

152 + buffer: DecodedText::default(),

153 + emitted_tool_count: 0,

154 + };

155 + parser.reset_to_implicit();

156 + parser

157 + }

158 +

159 + /// Clear all per-stream state.

160 + fn reset_state(&mut self) {

161 + self.buffer.clear();

162 + self.closed.fill(false);

163 + self.strip = self.template.regions.iter().map(StripState::new).collect();

164 + self.emitted_tool_count = 0;

165 + self.reset_to_implicit();

166 + }

167 +

168 + /// Return to the implicit region, or discard text when there is none.

169 + fn reset_to_implicit(&mut self) {

170 + self.mode = match self.template.implicit {

171 + Some(region) => {

172 + HfMode::Implicit(self.occurrence(region, false, Captures::new(), String::new()))

173 + }

174 + None => HfMode::Discard,

175 + };

176 + }

177 +

178 + /// Start a new occurrence of `region`.

179 + fn occurrence(

180 + &mut self,

181 + region: usize,

182 + opened: bool,

183 + captures: Captures,

184 + raw_open: String,

185 + ) -> Occurrence {

186 + let spec = &self.template.regions[region];

187 + let mut separator = None;

188 + if let RegionKind::Text(text) = &spec.kind {

189 + if let Repeat::Join(join) = &text.repeat

190 + && !join.is_empty()

191 + && self.closed[region]

192 + {

193 + separator = Some(join.clone());

194 + }

195 + if text.repeat != Repeat::Once {

196 + self.strip[region] = StripState::new(spec);

197 + }

198 + }

199 + Occurrence {

200 + region,

201 + opened,

202 + captures,

203 + raw_open,

204 + body: String::new(),

205 + active_tool_index: None,

206 + separator,

207 + }

208 + }

209 +

210 + /// Parse buffered input until more is needed.

211 + fn drive(&mut self, output: &mut UnifiedParserOutput, phase: Phase) -> Result<()> {

212 + let template = Arc::clone(&self.template);

213 + loop {

214 + let watch = match &self.mode {

215 + HfMode::Explicit(occurrence) => &template.regions[occurrence.region].close_watch,

216 + HfMode::Discard | HfMode::Implicit(_) => &template.idle_watch,

217 + };

218 + let eof = matches!(phase, Phase::Eof);

219 + let Some((step, consumed_len)) = parse_buffered_event(&self.buffer.text, |input| {

220 + parse_next_event(input, &template, watch, eof)

221 + })?

222 + else {

223 + return Ok(());

224 + };

225 + let piece = self.buffer.drain_prefix(consumed_len);

226 + match step {

227 + HfEvent::Text => self.route(piece, output, phase),

228 + HfEvent::Boundary { watch, captures } => {

229 + self.close_current(output, phase)?;

230 + match watch.kind {

231 + WatchKind::Open => {

232 + self.open_explicit(watch.region, captures, piece.text, output, phase)

233 + }

234 + // The implicit region's close: start a fresh implicit occurrence.

235 + WatchKind::Close => {}

236 + }

237 + }

238 + }

239 + }

240 + }

241 +

242 + /// Route a text piece into the current region.

243 + fn route(&mut self, piece: DecodedText, output: &mut UnifiedParserOutput, phase: Phase) {

244 + let (occurrence, region) = match &mut self.mode {

245 + HfMode::Discard => return,

246 + HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) => {

247 + let region = &self.template.regions[occurrence.region];

248 + (occurrence, region)

249 + }

250 + };

251 + let RegionKind::Text(text) = &region.kind else {

252 + if !piece.text.is_empty() {

253 + occurrence.opened = true;

254 + occurrence.body.push_str(&piece.text);

255 + }

256 + return;

257 + };

258 + let reasoning = text.role == TextRole::Reasoning;

259 + if piece.text.is_empty() {

260 + // Zero-width tokens still count as reasoning.

261 + if reasoning && phase.emits() {

262 + output.push_reasoning(piece);

263 + }

264 + return;

265 + }

266 + occurrence.opened = true;

267 +

268 + let strip = &mut self.strip[occurrence.region];

269 + if !phase.emits() {

270 + // Prompt text is never emitted; it only decides whether later

271 + // leading whitespace is still stripped.

272 + strip.started |= !piece.text.trim_start().is_empty();

273 + return;

274 + }

275 + let (visible, dropped) = strip.feed(piece);

276 + if reasoning {

277 + push_dropped_reasoning(output, dropped);

278 + }

279 + if visible.text.is_empty() {

280 + return;

281 + }

282 + if let Some(separator) = occurrence.separator.take() {

283 + push_text(output, text.role, DecodedText::unattributed(separator));

284 + }

285 + push_text(output, text.role, visible);

286 + }

287 +

288 + /// Open an explicit region.

289 + fn open_explicit(

290 + &mut self,

291 + region: usize,

292 + captures: Captures,

293 + raw_open: String,

294 + output: &mut UnifiedParserOutput,

295 + phase: Phase,

296 + ) {

297 + let mut occurrence = self.occurrence(region, true, captures, raw_open);

298 + let opener_name = match &self.template.regions[region].kind {

299 + RegionKind::ToolCalls(ToolCallRegion {

300 + name_group: Some(group),

301 + ..

302 + }) => occurrence.captures.get(group).and_then(Value::as_str).map(str::to_string),

303 + _ => None,

304 + };

305 + if let Some(name) = opener_name

306 + && phase.emits()

307 + {

308 + let tool_index = self.allocate_tool_index();

309 + occurrence.active_tool_index = Some(tool_index);

310 + output.push_call(ToolCallDelta {

311 + tool_index,

312 + name: Some(name),

313 + arguments: String::new(),

314 + });

315 + }

316 + self.mode = HfMode::Explicit(occurrence);

317 + }

318 +

319 + /// Close the current region and reset to the implicit region.

320 + ///

321 + /// Skipped (aside from the reset) when the current region never opened --

322 + /// avoids vacuous open/close pairs at every explicit boundary.

323 + fn close_current(&mut self, output: &mut UnifiedParserOutput, phase: Phase) -> Result<()> {

324 + let template = Arc::clone(&self.template);

325 + let mode = std::mem::replace(&mut self.mode, HfMode::Discard);

326 + let occurrence = match mode {

327 + HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) if occurrence.opened => {

328 + Some(occurrence)

329 + }

330 + _ => None,

331 + };

332 + // Without `repeats`, trailing whitespace stays held in case the field

333 + // continues; it is dropped at the end of the stream.

334 + let held = occurrence

335 + .as_ref()

336 + .filter(|occurrence| {

337 + matches!(&template.regions[occurrence.region].kind,

338 + RegionKind::Text(text) if text.repeat != Repeat::Once)

339 + })

340 + .map(|occurrence| take(&mut self.strip[occurrence.region].held));

341 + if let Some(occurrence) = &occurrence {

342 + self.closed[occurrence.region] = true;

343 + }

344 + self.reset_to_implicit();

345 + // Regions closed inside the prompt contribute nothing to the output.

346 + let Some(occurrence) = occurrence.filter(|_| phase.emits()) else {

347 + return Ok(());

348 + };

349 +

350 + let region = &template.regions[occurrence.region];

351 + match &region.kind {

352 + RegionKind::Text(text) => {

353 + if text.role == TextRole::Reasoning

354 + && let Some(held) = held

355 + {

356 + push_dropped_reasoning(output, held);

357 + }

358 + // An empty occurrence still joins: `previous + join + ""`.

359 + if let Some(separator) = occurrence.separator {

360 + push_text(output, text.role, DecodedText::unattributed(separator));

361 + }

362 + }

363 + RegionKind::ToolCalls(calls) => {

364 + let parsed = calls.content.parse(&occurrence.body)?;

365 + let value = match &calls.transform {

366 + Some(transform) => {

367 + transform.apply(region.name.as_str(), parsed, &occurrence.captures)?

368 + }

369 + None => parsed,

370 + };

371 + self.emit_calls(value, occurrence.active_tool_index, output)?;

372 + }

373 + }

374 + Ok(())

375 + }

376 +

377 + /// Emit the tool calls of a closed tool-call region.

378 + // TODO: stream arguments incrementally instead of once at region close; the

379 + // dialects (e.g. Gemma 4's unquoted keys and `<|"|>` strings, `xml-inline`)

380 + // need incremental conversion rather than raw passthrough.

381 + fn emit_calls(

382 + &mut self,

383 + value: Value,

384 + started: Option<usize>,

385 + output: &mut UnifiedParserOutput,

386 + ) -> Result<()> {

387 + let calls = match value {

388 + Value::Array(calls) => calls,

389 + call => vec![call],

390 + };

391 + for (position, call) in calls.into_iter().enumerate() {

392 + let (name, arguments) = call_parts(call, &self.tool_schemas)?;

393 + let (tool_index, name) = match (position, started) {

394 + (0, Some(tool_index)) => (tool_index, None),

395 + _ => (self.allocate_tool_index(), Some(name)),

396 + };

397 + output.push_call(ToolCallDelta {

398 + tool_index,

399 + name,

400 + arguments,

401 + });

402 + }

403 + Ok(())

404 + }

405 +

406 + fn allocate_tool_index(&mut self) -> usize {

407 + let tool_index = self.emitted_tool_count;

408 + self.emitted_tool_count += 1;

409 + tool_index

410 + }

411 +

412 + /// Return the prompt text after the last start-anchor match, if any.

413 + ///

414 + /// Searches geometrically growing tail windows before decoding the whole

415 + /// prompt. A decoded suffix can differ from the full decode only in its first

416 + /// character, so a match is accepted only after that character unless the

417 + /// window is the whole prompt.

418 + fn prompt_remainder(&self, prompt_token_ids: &[u32]) -> Result<Option<String>> {

419 + let len = prompt_token_ids.len();

420 + let windows = PROMPT_WINDOWS.into_iter().filter(|window| *window < len).chain([len]);

421 + for window in windows {

422 + let text = self.tokenizer.decode(&prompt_token_ids[len - window..], false).map_err(

423 + |error| parsing_failed!("failed to decode prompt: {}", error.as_report()),

424 + )?;

425 + let whole = window == len;

426 + if let Some((start, end)) = self.template.last_start_anchor(&text) {

427 + let first_char = text.chars().next().map_or(0, char::len_utf8);

428 + if whole || start >= first_char {

429 + return Ok(Some(text[end..].to_string()));

430 + }

431 + }

432 + }

433 + Ok(None)

434 + }

435 +}

436 +

437 +impl StripState {

438 + fn new(region: &Region) -> Self {

439 + Self {

440 + enabled: matches!(

441 + region.kind,

442 + RegionKind::Text(TextRegion { strip: true, .. })

443 + ),

444 + ..Self::default()

445 + }

446 + }

447 +

448 + /// Split a piece into its visible part and whitespace dropped by `strip`.

449 + ///

450 + /// Leading whitespace is dropped until the first non-whitespace character;

451 + /// trailing whitespace is held until more non-whitespace text arrives (then

452 + /// released) or the region closes (then dropped).

453 + fn feed(&mut self, mut piece: DecodedText) -> (DecodedText, DecodedText) {

454 + if !self.enabled {

455 + return (piece, DecodedText::default());

456 + }

457 + let mut dropped = DecodedText::default();

458 + if !self.started {

459 + let lead = piece.text.len() - piece.text.trim_start().len();

460 + if lead == piece.text.len() {

461 + return (DecodedText::default(), piece);

462 + }

463 + dropped = piece.drain_prefix(lead);

464 + self.started = true;

465 + }

466 + let body_len = piece.text.trim_end().len();

467 + if body_len == 0 {

468 + self.held.append(piece);

469 + return (DecodedText::default(), dropped);

470 + }

471 + let head = piece.drain_prefix(body_len);

472 + let mut visible = take(&mut self.held);

473 + visible.append(head);

474 + self.held = piece;

475 + (visible, dropped)

476 + }

477 +}

478 +

479 +/// Push visible text of a text region.

480 +fn push_text(output: &mut UnifiedParserOutput, role: TextRole, piece: DecodedText) {

481 + match role {

482 + TextRole::Content => output.push_text(piece.text),

483 + TextRole::Reasoning => output.push_reasoning(piece),

484 + }

485 +}

486 +

487 +/// Keep the tokens of stripped reasoning whitespace in the reasoning count.

488 +fn push_dropped_reasoning(output: &mut UnifiedParserOutput, dropped: DecodedText) {

489 + if dropped.attributions.is_empty() {

490 + return;

491 + }

492 + output.push_reasoning(DecodedText {

493 + text: String::new(),

494 + attributions: dropped

495 + .attributions

496 + .into_iter()

497 + .map(|attribution| TokenAttribution {

498 + token_id: attribution.token_id,

499 + anchor: TokenAnchor::ZeroWidth { byte_offset: 0 },

500 + })

501 + .collect(),

502 + });

503 +}

504 +

505 +/// One tool call produced by a tool-call region, after its transform.

506 +#[derive(Deserialize)]

507 +#[serde(untagged)]

508 +enum ToolCallValue {

509 + /// The OpenAI shape `{"type": "function", "function": {...}}`.

510 + Wrapped { function: FunctionValue },

511 + /// The bare `{"name", "arguments"}` shape.

512 + Bare(FunctionValue),

513 +}

514 +

515 +#[derive(Deserialize)]

516 +struct FunctionValue {

517 + name: String,

518 + /// Missing or `null` arguments become `{}`.

519 + #[serde(default)]

520 + arguments: Option<ArgumentsValue>,

521 +}

522 +

523 +#[derive(Deserialize)]

524 +#[serde(untagged)]

525 +enum ArgumentsValue {

526 + /// An argument object, typed by the tool schema.

527 + Object(Map<String, Value>),

528 + /// Pre-serialized or non-JSON arguments (e.g. an `allow_non_json`

529 + /// fallback), passed through unchanged.

530 + Text(String),

531 + /// Any other JSON value, serialized as is.

532 + Other(Value),

533 +}

534 +

535 +/// Extract the function name and serialized arguments of one tool-call value.

536 +///

537 +/// String argument values are converted by the calling tool's parameter

538 +/// schema, as for the other tool parsers.

539 +fn call_parts(call: Value, tool_schemas: &ToolSchemas) -> Result<(String, String)> {

540 + let (ToolCallValue::Wrapped { function } | ToolCallValue::Bare(function)) =

541 + ToolCallValue::deserialize(&call).map_err(|_| {

542 + parsing_failed!(

543 + "tool call must be {{\"function\": {{\"name\": ..., \"arguments\": ...}}}} or \

544 + {{\"name\": ..., \"arguments\": ...}} with a string name, got {call}"

545 + )

546 + })?;

547 + let FunctionValue { name, arguments } = function;

548 + let arguments = match arguments {

549 + None => "{}".to_string(),

550 + Some(ArgumentsValue::Text(arguments)) => arguments,

551 + Some(ArgumentsValue::Object(mut arguments)) => {

552 + convert_text_arguments(tool_schemas, &name, &mut arguments);

553 + serialize_arguments(&arguments)?

554 + }

555 + Some(ArgumentsValue::Other(arguments)) => serialize_arguments(&arguments)?,

556 + };

557 + Ok((name, arguments))

558 +}

559 +

560 +/// Convert the argument values a content parser left as raw text by the tool

561 +/// schema: top-level strings, and string lists collected from duplicate keys

562 +/// (`merge_duplicates`).

563 +///

564 +/// Deeper values are the content parser's own JSON and are kept, as in

565 +/// Transformers' `_coerce_tool_calls`.

566 +fn convert_text_arguments(

567 + tool_schemas: &ToolSchemas,

568 + function_name: &str,

569 + arguments: &mut Map<String, Value>,

570 +) {

571 + for (name, value) in arguments.iter_mut() {

572 + let values = match value {

573 + Value::Array(items) => items.as_mut_slice(),

574 + value => std::slice::from_mut(value),

575 + };

576 + for value in values {

577 + if let Value::String(text) = value {

578 + *value = tool_schemas.convert_param_with_schema(function_name, name, take(text));

579 + }

580 + }

581 + }

582 +}

583 +

584 +/// Serialize tool-call arguments to JSON text.

585 +fn serialize_arguments(arguments: &impl serde::Serialize) -> Result<String> {

586 + serde_json::to_string(arguments).map_err(|error| {

587 + parsing_failed!("failed to serialize tool arguments: {}", error.as_report())

588 + })

589 +}

590 +

591 +/// Parse one event: safe text before the next marker, or a boundary at it.

592 +fn parse_next_event(

593 + input: &mut Partial<&str>,

594 + template: &ResponseTemplate,

595 + watch: &WatchSet,

596 + eof: bool,

597 +) -> ModalResult<HfEvent> {

598 + let markers: Vec<&str> = watch.markers.iter().map(String::as_str).collect();

599 + match safe_text_len_mul(input, &markers) {

600 + Ok(_) => return Ok(HfEvent::Text),

601 + // A marker, complete or partial, starts here (or the input is empty).

602 + Err(ErrMode::Incomplete(_)) => {}

603 + Err(error) => return Err(error),

604 + }

605 +

606 + // Any pending boundary blocks the position; otherwise the preferred match

607 + // wins. At the end of the stream, a partial marker matches nothing and is

608 + // released as text one character at a time.

609 + let text = **input;

610 + let mut best: Option<BoundaryMatch> = None;

611 + for &watch in &watch.boundaries {

612 + match template.boundary(watch).resolve(text, eof) {

613 + Resolution::Pending => return incomplete(),

614 + Resolution::NoMatch => {}

615 + Resolution::Matched { len, captures } => {

616 + let candidate = BoundaryMatch {

617 + watch,

618 + len,

619 + captures,

620 + };

621 + if best.as_ref().is_none_or(|best| candidate.key(template) < best.key(template)) {

622 + best = Some(candidate);

623 + }

624 + }

625 + }

626 + }

627 +

628 + match best {

629 + Some(BoundaryMatch {

630 + watch,

631 + len,

632 + captures,

633 + }) => {

634 + input.next_slice(len);

635 + Ok(HfEvent::Boundary { watch, captures })

636 + }

637 + None => {

638 + // Not a boundary: the marker's first character is plain text.

639 + let len = text.chars().next().map_or(0, char::len_utf8);

640 + input.next_slice(len);

641 + Ok(HfEvent::Text)

642 + }

643 + }

644 +}

645 +

646 +impl UnifiedParser for HfUnifiedParser {

647 + fn create(_tools: &[Tool], _tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>

648 + where

649 + Self: Sized + 'static,

650 + {

651 + Err(UnifiedParserError::NoNamedConstructor {

652 + parser: "hf",

653 + built_from: "the checkpoint's response_template",

654 + })

655 + }

656 +

657 + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {

658 + self.reset_state();

659 + if prompt_token_ids.is_empty() {

660 + return Ok(());

661 + }

662 + let Some(remainder) = self.prompt_remainder(prompt_token_ids)? else {

663 + self.template.warn_missing_anchor_once();

664 + return Ok(());

665 + };

666 + // A partial boundary at the end of the prompt stays buffered, so the

667 + // generated text can complete it.

668 + self.buffer = DecodedText::unattributed(remainder);

669 + self.drive(&mut UnifiedParserOutput::default(), Phase::Prefill)

670 + }

671 +

672 + fn preserve_special_tokens(&self) -> bool {

673 + // Templates spell boundaries with special tokens and expect decoding

674 + // with them kept.

675 + true

676 + }

677 +

678 + // TODO: derive a structural-tag output grammar from literal-bounded templates.

679 +

680 + fn parse_into(&mut self, delta: DecodedText, output: &mut UnifiedParserOutput) -> Result<()> {

681 + self.buffer.append(delta);

682 + self.drive(output, Phase::Stream)

683 + }

684 +

685 + fn finish(&mut self) -> Result<UnifiedParserOutput> {

686 + let mut output = UnifiedParserOutput::default();

687 + self.drive(&mut output, Phase::Eof)?;

688 + // Trailing zero-width tokens carry no text for the step loop to consume.

689 + let rest = self.buffer.take();

690 + self.route(rest, &mut output, Phase::Eof);

691 + self.close_current(&mut output, Phase::Eof)?;

692 + for (region, strip) in self.template.regions.iter().zip(&mut self.strip) {

693 + let held = take(&mut strip.held);

694 + if matches!(&region.kind, RegionKind::Text(text) if text.role == TextRole::Reasoning) {

695 + push_dropped_reasoning(&mut output, held);

696 + }

697 + }

698 +

699 + let missing: Vec<_> = self

700 + .template

701 + .regions

702 + .iter()

703 + .zip(&self.closed)

704 + .filter(|(region, closed)| !region.optional && !**closed)

705 + .map(|(region, _)| region.name.as_str())

706 + .collect();

707 + if !missing.is_empty() {

708 + return Err(parsing_failed!(

709 + "Required response_template fields missing from parsed output: {missing:?}"

710 + ));

711 + }

712 +

713 + self.reset_state();

714 + Ok(output)

715 + }

716 +

717 + fn reset(&mut self) -> String {

718 + let mut raw = String::new();

719 + if let HfMode::Implicit(occurrence) | HfMode::Explicit(occurrence) =

720 + std::mem::replace(&mut self.mode, HfMode::Discard)

721 + {

722 + raw.push_str(&occurrence.raw_open);

723 + raw.push_str(&occurrence.body);

724 + raw.push_str(&self.strip[occurrence.region].held.text);

725 + }

726 + raw.push_str(&self.buffer.take().text);

727 + self.reset_state();

728 + raw

729 + }

730 +}

rust/src/parser/src/unified/hf/pattern.rs new

+403

@@ -0,0 +1,403 @@

1 +// SPDX-License-Identifier: Apache-2.0

2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +//! Streaming resolution of `open_pattern` / `close_pattern` boundaries.

5 +//!

6 +//! Transformers relies on the Python `regex` module's `partial=True` search to

7 +//! decide whether a match at the end of the buffer is final, could still appear,

8 +//! or is impossible. The Rust `regex` crate has no partial-match API, so this

9 +//! module answers that question by stepping a `regex-automata` DFA, and uses a

10 +//! `meta::Regex` for the actual match span and captures.

11 +

12 +use regex_automata::dfa::{Automaton, StartKind, dense};

13 +use regex_automata::meta::Regex;

14 +use regex_automata::nfa::thompson::{self, WhichCaptures};

15 +use regex_automata::util::primitives::StateID;

16 +use regex_automata::{Anchored, Input, MatchKind, PatternID};

17 +use regex_syntax::hir::literal::{ExtractKind, Extractor};

18 +use regex_syntax::hir::{Hir, HirKind};

19 +use serde_json::{Map, Value};

20 +use thiserror_ext::AsReport as _;

21 +

22 +use super::{Result, unsupported};

23 +

24 +/// Upper bound for the memory of one boundary DFA.

25 +const DFA_SIZE_LIMIT: usize = 16 << 20;

26 +

27 +/// Named capture groups of a boundary match, as string values: the scope a

28 +/// transform template reads them from.

29 +pub(super) type Captures = Map<String, Value>;

30 +

31 +/// Outcome of resolving a boundary at a candidate position.

32 +#[derive(Debug, Clone, PartialEq, Eq)]

33 +pub(super) enum Resolution {

34 + /// The boundary matches `len` bytes; more input cannot change the match.

35 + Matched { len: usize, captures: Captures },

36 + /// More input could still produce or change a match.

37 + Pending,

38 + /// No match starts here, whatever input follows.

39 + NoMatch,

40 +}

41 +

42 +/// A regex boundary that can be resolved against a growing buffer.

43 +#[derive(Debug, Clone)]

44 +pub(super) struct StreamingPattern {

45 + /// Finite set of non-empty literals, one of which starts every match.

46 + prefixes: Vec<String>,

47 + /// Anchored leftmost-first DFA of the pattern: decides whether the

48 + /// leftmost-first match is final. Unicode word boundaries are handled

49 + /// heuristically, quitting on non-ASCII bytes.

50 + first: dense::DFA<Vec<u32>>,

51 + /// Anchored DFA with every look-around assertion removed and

52 + /// `MatchKind::All`, used when `first` quits. Relaxing an assertion only

53 + /// enlarges the language, so once this DFA is dead no continuation can

54 + /// produce a later-ending match of the real pattern. `MatchKind::All` is

55 + /// required for that argument: leftmost-first pruning could discard the

56 + /// thread the real pattern takes when an assertion fails (`a\b|ab` on `ab`).

57 + fallback: Option<dense::DFA<Vec<u32>>>,

58 + /// The pattern itself, for the match span and named captures.

59 + exact: Regex,

60 +}

61 +

62 +/// Result of walking a DFA over the available bytes.

63 +enum Walk {

64 + /// No continuation can change the result.

65 + Final,

66 + /// More input could still change the result.

67 + Alive,

68 + /// The DFA gave up (non-ASCII byte near a Unicode word boundary).

69 + Quit,

70 +}

71 +

72 +impl StreamingPattern {

73 + /// Compile a boundary pattern in Python `regex` syntax, with `re.DOTALL`.

74 + ///

75 + /// Streaming resolution only starts at occurrences of the pattern's literal

76 + /// prefixes, so every match must start with one of a finite set of non-empty

77 + /// literals.

78 + pub fn new(scope: &str, pattern: &str) -> Result<Self> {

79 + let hir = regex_syntax::ParserBuilder::new()

80 + .dot_matches_new_line(true)

81 + .build()

82 + .parse(pattern)

83 + .map_err(|error| {

84 + unsupported!(

85 + "{scope}: regex {pattern:?} is not supported by the Rust regex engine: {}",

86 + error.as_report()

87 + )

88 + })?;

89 +

90 + let prefixes = literal_prefixes(&hir).ok_or_else(|| {

91 + unsupported!(

92 + "{scope}: pattern {pattern:?} must start with a literal prefix for streaming \

93 + matching (e.g. `<tool_call>` in `<tool_call>(?P<name>\\w+)`)"

94 + )

95 + })?;

96 +

97 + let first = build_dfa(scope, pattern, &hir, MatchKind::LeftmostFirst)?;

98 + let fallback = hir

99 + .properties()

100 + .look_set()

101 + .contains_word_unicode()

102 + .then(|| build_dfa(scope, pattern, &relax_assertions(&hir), MatchKind::All))

103 + .transpose()?;

104 + let exact = Regex::builder().build_from_hir(&hir).map_err(|error| {

105 + unsupported!(

106 + "{scope}: failed to compile regex {pattern:?}: {}",

107 + error.as_report()

108 + )

109 + })?;

110 +

111 + Ok(Self {

112 + prefixes,

113 + first,

114 + fallback,

115 + exact,

116 + })

117 + }

118 +

119 + /// Literals, one of which starts every match.

120 + pub fn prefixes(&self) -> &[String] {

121 + &self.prefixes

122 + }

123 +

124 + /// Named capture groups of the pattern.

125 + pub fn group_names(&self) -> impl Iterator<Item = &str> {

126 + self.exact.group_info().pattern_names(PatternID::ZERO).flatten()

127 + }

128 +

129 + /// Resolve a match starting at the beginning of `text`.

130 + ///

131 + /// `eof` means no more input will arrive.

132 + pub fn resolve(&self, text: &str, eof: bool) -> Resolution {

133 + let walk = match walk(&self.first, text) {

134 + Walk::Quit => walk(

135 + self.fallback.as_ref().expect("quits imply a fallback"),

136 + text,

137 + ),

138 + walk => walk,

139 + };

140 + if matches!(walk, Walk::Alive) && !eof {

141 + return Resolution::Pending;

142 + }

143 +

144 + let input = Input::new(text).anchored(Anchored::Yes);

145 + let mut captures = self.exact.create_captures();

146 + self.exact.search_captures(&input, &mut captures);

147 + let Some(found) = captures.get_match() else {

148 + return Resolution::NoMatch;

149 + };

150 + let names = self.exact.group_info().pattern_names(PatternID::ZERO);

151 + let captures = names

152 + .enumerate()

153 + .filter_map(|(index, name)| {

154 + let span = captures.get_group(index)?;

155 + Some((

156 + name?.to_string(),

157 + Value::String(text[span.range()].to_string()),

158 + ))

159 + })

160 + .collect();

161 + Resolution::Matched {

162 + len: found.end(),

163 + captures,

164 + }

165 + }

166 +}

167 +

168 +/// Walk `dfa` from an anchored start over `text`.

169 +///

170 +/// The walk is final once the DFA is dead, or once it is in a match state whose

171 +/// transitions are all dead: that removes the one byte of holdback that delayed

172 +/// DFA match reporting would otherwise add. A quit transition counts as live.

173 +fn walk(dfa: &dense::DFA<Vec<u32>>, text: &str) -> Walk {

174 + let input = Input::new(text).anchored(Anchored::Yes);

175 + let Ok(mut state) = dfa.start_state_forward(&input) else {

176 + return Walk::Quit;

177 + };

178 + for byte in text.bytes() {

179 + state = dfa.next_state(state, byte);

180 + if dfa.is_quit_state(state) {

181 + return Walk::Quit;

182 + }

183 + if dfa.is_dead_state(state) || is_terminal_match(dfa, state) {

184 + return Walk::Final;

185 + }

186 + }

187 + Walk::Alive

188 +}

189 +

190 +/// Return whether `state` reports a match and cannot continue.

191 +fn is_terminal_match(dfa: &dense::DFA<Vec<u32>>, state: StateID) -> bool {

192 + dfa.is_match_state(state)

193 + && (0..=u8::MAX).all(|byte| dfa.is_dead_state(dfa.next_state(state, byte)))

194 +}

195 +

196 +/// Build an anchored dense DFA for `hir`.

197 +fn build_dfa(

198 + scope: &str,

199 + pattern: &str,

200 + hir: &Hir,

201 + kind: MatchKind,

202 +) -> Result<dense::DFA<Vec<u32>>> {

203 + let nfa = thompson::Compiler::new()

204 + .configure(thompson::Config::new().which_captures(WhichCaptures::None))

205 + .build_from_hir(hir)

206 + .map_err(|error| {

207 + unsupported!(

208 + "{scope}: failed to compile regex {pattern:?}: {}",

209 + error.as_report()

210 + )

211 + })?;

212 + dense::Builder::new()

213 + .configure(

214 + dense::Config::new()

215 + .match_kind(kind)

216 + .start_kind(StartKind::Anchored)

217 + .unicode_word_boundary(true)

218 + .dfa_size_limit(Some(DFA_SIZE_LIMIT))

219 + .determinize_size_limit(Some(DFA_SIZE_LIMIT)),

220 + )

221 + .build_from_nfa(&nfa)

222 + .map_err(|error| {

223 + unsupported!(

224 + "{scope}: regex {pattern:?} cannot be matched incrementally: {}",

225 + error.as_report()

226 + )

227 + })

228 +}

229 +

230 +/// Replace every look-around assertion with the empty regex.

231 +fn relax_assertions(hir: &Hir) -> Hir {

232 + match hir.kind() {

233 + HirKind::Look(_) => Hir::empty(),

234 + HirKind::Repetition(repetition) => {

235 + let mut repetition = repetition.clone();

236 + repetition.sub = Box::new(relax_assertions(&repetition.sub));

237 + Hir::repetition(repetition)

238 + }

239 + HirKind::Capture(capture) => {

240 + let mut capture = capture.clone();

241 + capture.sub = Box::new(relax_assertions(&capture.sub));

242 + Hir::capture(capture)

243 + }

244 + HirKind::Concat(subs) => Hir::concat(subs.iter().map(relax_assertions).collect()),

245 + HirKind::Alternation(subs) => Hir::alternation(subs.iter().map(relax_assertions).collect()),

246 + HirKind::Empty | HirKind::Literal(_) | HirKind::Class(_) => hir.clone(),

247 + }

248 +}

249 +

250 +/// Extract the finite set of non-empty literal prefixes starting every match.

251 +fn literal_prefixes(hir: &Hir) -> Option<Vec<String>> {

252 + let seq = Extractor::new().kind(ExtractKind::Prefix).extract(hir);

253 + let mut prefixes: Vec<String> = Vec::new();

254 + for literal in seq.literals()? {

255 + let bytes = literal.as_bytes();

256 + // Truncated inexact literals may end inside a UTF-8 sequence; any valid

257 + // prefix of a prefix is still a prefix.

258 + let valid = match std::str::from_utf8(bytes) {

259 + Ok(text) => text,

260 + Err(error) => std::str::from_utf8(&bytes[..error.valid_up_to()]).ok()?,

261 + };

262 + if valid.is_empty() {

263 + return None;

264 + }

265 + if !prefixes.iter().any(|prefix| prefix == valid) {

266 + prefixes.push(valid.to_string());

267 + }

268 + }

269 + (!prefixes.is_empty()).then_some(prefixes)

270 +}

271 +

272 +#[cfg(test)]

273 +mod tests {

274 + use super::*;

275 +

276 + const MUSE_INVOKE: &str = r#"<atem:invoke\b[^>]*?\bname="(?P<name>[^"]+)">"#;

277 + const GEMMA4_CALL: &str = r"<\|tool_call>call:(?P<name>\w+)";

278 + const GPT_OSS_CALL: &str =

279 + r"<\|channel\|>commentary to=functions\.(?P<name>\w+).*?<\|message\|>";

280 +

281 + fn pattern(source: &str) -> StreamingPattern {

282 + StreamingPattern::new("test", source).unwrap()

283 + }

284 +

285 + fn matched(len: usize, name: &str) -> Resolution {

286 + Resolution::Matched {

287 + len,

288 + captures: Captures::from_iter([("name".to_string(), name.into())]),

289 + }

290 + }

291 +

292 + #[test]

293 + fn extracts_literal_prefixes() {

294 + assert_eq!(pattern(GEMMA4_CALL).prefixes(), ["<|tool_call>call:"]);

295 + assert_eq!(pattern(MUSE_INVOKE).prefixes(), ["<atem:invoke"]);

296 + assert_eq!(

297 + pattern(r"to=(?:user|commentary)<\|message\|>").prefixes(),

298 + ["to=user<|message|>", "to=commentary<|message|>"]

299 + );

300 + assert_eq!(

301 + pattern(r"\n?</response>").prefixes(),

302 + ["\n</response>", "</response>"]

303 + );

304 + for source in [r"(?:^|<think>\s*)", r"(?:<\|m\|>)?[^<]*<\|c\|>", r".*x"] {

305 + let result = StreamingPattern::new("test", source);

306 + assert!(

307 + matches!(

308 + result,

309 + Err(super::super::HfTemplateError::Unsupported { .. })

310 + ),

311 + "{source}"

312 + );

313 + }

314 + }

315 +

316 + #[test]

317 + fn word_boundaries_follow_unicode_semantics() {

318 + let muse = pattern(MUSE_INVOKE);

319 + assert_eq!(

320 + muse.resolve(r#"<atem:invoke name="get">rest"#, false),

321 + matched(24, "get")

322 + );

323 + let unicode_attr = r#"<atem:invoke title="天气" name="获取">rest"#;

324 + assert_eq!(

325 + muse.resolve(unicode_attr, false),

326 + matched(unicode_attr.len() - 4, "获取")

327 + );

328 + assert_eq!(

329 + muse.resolve(r#"<atem:invoke name="get"#, false),

330 + Resolution::Pending

331 + );

332 + assert_eq!(

333 + muse.resolve(r#"<atem:invoker name="x">r"#, false),

334 + Resolution::NoMatch

335 + );

336 + assert_eq!(

337 + muse.resolve(r#"<atem:invoke xname="x">r"#, false),

338 + Resolution::NoMatch

339 + );

340 + assert_eq!(

341 + muse.resolve(r#"<atem:invoke é name="x">r"#, false),

342 + matched(25, "x")

343 + );

344 + assert_eq!(

345 + muse.resolve(r#"<atem:invoke éname="x">r"#, false),

346 + Resolution::NoMatch

347 + );

348 + assert_eq!(

349 + muse.resolve("<atem:invoke 使用标签", false),

350 + Resolution::Pending

351 + );

352 + assert_eq!(

353 + muse.resolve("<atem:invoke 使用标签", true),

354 + Resolution::NoMatch

355 + );

356 + assert_eq!(

357 + muse.resolve("<atem:invoke 使用> 正文", false),

358 + Resolution::NoMatch

359 + );

360 +

361 + let alternation = pattern(r"a\b|ab");

362 + assert_eq!(

363 + alternation.resolve("ab!", true),

364 + Resolution::Matched {

365 + len: 2,

366 + captures: Captures::new()

367 + }

368 + );

369 + }

370 +

371 + #[test]

372 + fn greedy_and_lazy_tails_commit_like_transformers() {

373 + let gemma4 = pattern(GEMMA4_CALL);

374 + assert_eq!(

375 + gemma4.resolve("<|tool_call>call:get_wea", false),

376 + Resolution::Pending

377 + );

378 + assert_eq!(

379 + gemma4.resolve("<|tool_call>call:get_weather{", false),

380 + matched(28, "get_weather")

381 + );

382 + assert_eq!(

383 + gemma4.resolve("<|tool_call>call:天气{", false),

384 + matched(23, "天气")

385 + );

386 + assert_eq!(

387 + gemma4.resolve("<|tool_call>call:{", false),

388 + Resolution::NoMatch

389 + );

390 + assert_eq!(

391 + gemma4.resolve("<|tool_call>call:get", true),

392 + matched(20, "get")

393 + );

394 +

395 + let gpt_oss = pattern(GPT_OSS_CALL);

397 + assert_eq!(gpt_oss.resolve(opener, false), Resolution::Pending);

398 + assert_eq!(

399 + gpt_oss.resolve(&format!("{opener}{{\"a\""), false),

400 + matched(opener.len(), "get_weather")

401 + );

402 + }

403 +}

rust/src/parser/src/unified/hf/spec.rs new

+202

@@ -0,0 +1,202 @@

1 +// SPDX-License-Identifier: Apache-2.0

2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +//! Serde mirror of the `response_template` JSON.

5 +//!

6 +//! Key sets and defaults follow `response_templates.py`; semantic validation

7 +//! happens when compiling into [`super::ResponseTemplate`].

8 +

9 +use serde::Deserialize;

10 +use serde::de::DeserializeOwned;

11 +use serde_json::{Map, Value};

12 +use serde_with::serde_as;

13 +use thiserror_ext::AsReport as _;

14 +

15 +use super::{Result, invalid};

16 +

17 +/// Top-level `response_template` object.

18 +#[serde_as]

19 +#[derive(Debug, Clone, Deserialize)]

20 +#[serde(deny_unknown_fields)]

21 +pub(super) struct TemplateSpec {

22 + #[serde(default = "default_version")]

23 + version: u64,

24 + /// Constant keys of the Transformers output dict (e.g. `role`). Unused by the

25 + /// event-based executor, but validated as part of the schema.

26 + #[serde(default, rename = "defaults")]

27 + _defaults: Map<String, Value>,

28 + /// Field specs in declaration order.

29 + #[serde_as(as = "serde_with::Map<_, _>")]

30 + pub fields: Vec<(String, FieldSpec)>,

31 + start_anchor: Option<Literals>,

32 + start_anchor_pattern: Option<String>,

33 +}

34 +

35 +/// One entry of `fields`.

36 +#[derive(Debug, Clone, Deserialize)]

37 +#[serde(deny_unknown_fields)]

38 +pub(super) struct FieldSpec {

39 + open: Option<Literals>,

40 + open_pattern: Option<String>,

41 + close: Option<Literals>,

42 + close_pattern: Option<String>,

43 + #[serde(default)]

44 + pub content: ContentKind,

45 + /// Arguments of the `content` parser; their keys depend on the parser.

46 + #[serde(default)]

47 + pub content_args: Map<String, Value>,

48 + #[serde(default)]

49 + pub repeats: bool,

50 + pub join: Option<String>,

51 + #[serde(default = "default_optional")]

52 + pub optional: bool,

53 + /// A free-form JSON template, compiled into a [`super::transform::Transform`].

54 + pub transform: Option<Value>,

55 + #[serde(default)]

56 + pub transform_each: bool,

57 +}

58 +

59 +/// Name of a content parser.

60 +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]

61 +#[serde(rename_all = "kebab-case")]

62 +pub(super) enum ContentKind {

63 + #[default]

64 + Text,

65 + Int,

66 + Float,

67 + Bool,

68 + Json,

69 + XmlInline,

70 + KvLines,

71 +}

72 +

73 +/// A literal anchor: one string, or a list of alternatives.

74 +#[derive(Debug, Clone, Deserialize)]

75 +#[serde(untagged)]

76 +enum Literals {

77 + One(String),

78 + Many(Vec<String>),

79 +}

80 +

81 +/// An anchor after resolving its literal-or-pattern form.

82 +#[derive(Debug, Clone, PartialEq, Eq)]

83 +pub(super) enum AnchorSpec<'a> {

84 + /// Deduplicated literals in first-seen order.

85 + Literals(Vec<String>),

86 + /// A regex in Python `regex` syntax.

87 + Pattern(&'a str),

88 +}

89 +

90 +impl TemplateSpec {

91 + /// Parse the top-level template object.

92 + pub fn from_value(value: &Value) -> Result<Self> {

93 + let spec: Self = deserialize(value, "response_template")?;

94 + if spec.version != 1 {

95 + return Err(invalid!(

96 + "unsupported response_template version: {}",

97 + spec.version

98 + ));

99 + }

100 + if spec.fields.is_empty() {

101 + return Err(invalid!(

102 + "response_template.fields must be a non-empty dict"

103 + ));

104 + }

105 + Ok(spec)

106 + }

107 +

108 + /// The required start anchor.

109 + pub fn start_anchor(&self) -> Result<AnchorSpec<'_>> {

110 + anchor_spec(

111 + "response_template",

112 + (self.start_anchor.as_ref(), "start_anchor"),

113 + (self.start_anchor_pattern.as_deref(), "start_anchor_pattern"),

114 + )?

115 + .ok_or_else(|| {

116 + invalid!("response_template must define 'start_anchor' or 'start_anchor_pattern'.")

117 + })

118 + }

119 +}

120 +

121 +impl FieldSpec {

122 + /// The open anchor; `None` for the implicit field.

123 + pub fn open(&self, scope: &str) -> Result<Option<AnchorSpec<'_>>> {

124 + anchor_spec(

125 + scope,

126 + (self.open.as_ref(), "open"),

127 + (self.open_pattern.as_deref(), "open_pattern"),

128 + )

129 + }

130 +

131 + /// The close anchor; `None` when the field runs to the end of the stream.

132 + pub fn close(&self, scope: &str) -> Result<Option<AnchorSpec<'_>>> {

133 + anchor_spec(

134 + scope,

135 + (self.close.as_ref(), "close"),

136 + (self.close_pattern.as_deref(), "close_pattern"),

137 + )

138 + }

139 +}

140 +

141 +/// Deserialize `value`, reporting `context` and the JSON path of any error.

142 +pub(super) fn deserialize<T: DeserializeOwned>(value: &Value, context: &str) -> Result<T> {

143 + serde_path_to_error::deserialize(value).map_err(|error| {

144 + let path = error.path().to_string();

145 + let error = error.into_inner();

146 + if path == "." {

147 + invalid!("{context}: {}", error.as_report())

148 + } else {

149 + invalid!("{context}.{path}: {}", error.as_report())

150 + }

151 + })

152 +}

153 +

154 +/// Resolve an anchor given in literal form or regex form.

155 +///

156 +/// Mirrors `_compile_anchor`: both forms at once is an error, literal lists must be

157 +/// non-empty and contain no empty strings, and duplicates are dropped in first-seen

158 +/// order.

159 +fn anchor_spec<'a>(

160 + scope: &str,

161 + (literals, literal_key): (Option<&Literals>, &str),

162 + (pattern, pattern_key): (Option<&'a str>, &str),

163 +) -> Result<Option<AnchorSpec<'a>>> {

164 + match (literals, pattern) {

165 + (Some(_), Some(_)) => Err(invalid!(

166 + "{scope}: cannot specify both '{literal_key}' and '{pattern_key}'"

167 + )),

168 + (Some(literals), None) => {

169 + let raw = match literals {

170 + Literals::One(literal) => std::slice::from_ref(literal),

171 + Literals::Many(literals) => literals.as_slice(),

172 + };

173 + if raw.is_empty() {

174 + return Err(invalid!(

175 + "{scope}: '{literal_key}' list must contain at least one literal"

176 + ));

177 + }

178 + if raw.iter().any(String::is_empty) {

179 + return Err(invalid!(

180 + "{scope}: '{literal_key}' literals cannot be empty strings"

181 + ));

182 + }

183 + let mut deduped: Vec<String> = Vec::with_capacity(raw.len());

184 + for literal in raw {

185 + if !deduped.contains(literal) {

186 + deduped.push(literal.clone());

187 + }

188 + }

189 + Ok(Some(AnchorSpec::Literals(deduped)))

190 + }

191 + (None, Some(pattern)) => Ok(Some(AnchorSpec::Pattern(pattern))),

192 + (None, None) => Ok(None),

193 + }

194 +}

195 +

196 +fn default_version() -> u64 {

197 + 1

198 +}

199 +

200 +fn default_optional() -> bool {

201 + true

202 +}

rust/src/parser/src/unified/hf/template.rs new

+481

@@ -0,0 +1,481 @@

1 +// SPDX-License-Identifier: Apache-2.0

2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +//! Compiled `response_template`.

5 +//!

6 +//! Validation follows `response_templates.py`; the compiled form additionally

7 +//! maps fields to parser roles and precomputes each state's boundary markers.

8 +

9 +use std::fmt;

10 +use std::sync::atomic::{AtomicBool, Ordering};

11 +

12 +use regex_automata::meta::Regex;

13 +use serde_json::Value;

14 +

15 +use super::content::{ContentParser, TextArgs, compile_regex};

16 +use super::pattern::{Captures, Resolution, StreamingPattern};

17 +use super::spec::{AnchorSpec, FieldSpec, TemplateSpec};

18 +use super::transform::{FieldTransform, Transform};

19 +use super::{Result, invalid, unsupported};

20 +

21 +/// A compiled `response_template`, shared by all requests of one model.

22 +#[derive(Debug)]

23 +pub struct ResponseTemplate {

24 + /// Matches the start of the current assistant turn in the prompt.

25 + start_anchor: Regex,

26 + pub(super) regions: Vec<Region>,

27 + /// The field without an opener: the implicit-open / leftover sink.

28 + pub(super) implicit: Option<usize>,

29 + /// Boundaries watched outside explicit regions: every explicit open plus the

30 + /// implicit region's close.

31 + pub(super) idle_watch: WatchSet,

32 + warned_missing_anchor: AtomicBool,

33 +}

34 +

35 +/// One compiled field.

36 +#[derive(Debug)]

37 +pub(super) struct Region {

38 + pub name: FieldName,

39 + /// `None`: the implicit region.

40 + pub open: Option<Boundary>,

41 + /// `None`: the region runs to the end of the stream.

42 + pub close: Option<Boundary>,

43 + /// Boundaries watched inside this region: its close, if any.

44 + pub close_watch: WatchSet,

45 + pub optional: bool,

46 + pub kind: RegionKind,

47 +}

48 +

49 +/// A field this parser can report, in the lexical order of its name, which

50 +/// Transformers uses to break ties between boundaries.

51 +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]

52 +pub(super) enum FieldName {

53 + Content,

54 + ReasoningContent,

55 + Thinking,

56 + ToolCalls,

57 +}

58 +

59 +/// How a region's content is reported.

60 +#[derive(Debug)]

61 +pub(super) enum RegionKind {

62 + /// Streamed as it arrives.

63 + Text(TextRegion),

64 + /// Buffered, and parsed into tool calls at close.

65 + ToolCalls(ToolCallRegion),

66 +}

67 +

68 +#[derive(Debug)]

69 +pub(super) struct TextRegion {

70 + pub role: TextRole,

71 + /// Whether the value is stripped (`content_args.strip`).

72 + pub strip: bool,

73 + pub repeat: Repeat,

74 +}

75 +

76 +#[derive(Debug, Clone, Copy, PartialEq, Eq)]

77 +pub(super) enum TextRole {

78 + Content,

79 + Reasoning,

80 +}

81 +

82 +/// How repeated occurrences of a text field combine.

83 +#[derive(Debug, Clone, PartialEq, Eq)]

84 +pub(super) enum Repeat {

85 + /// No `repeats`: occurrences stream as one value.

86 + Once,

87 + /// `repeats` without `join`: each occurrence is its own value.

88 + Each,

89 + /// `repeats` with `join`: occurrences are joined by the separator.

90 + Join(String),

91 +}

92 +

93 +#[derive(Debug)]

94 +pub(super) struct ToolCallRegion {

95 + pub content: ContentParser,

96 + pub transform: Option<FieldTransform>,

97 + /// Opener capture group that provides the function name, allowing the tool

98 + /// call to start before its arguments are complete.

99 + pub name_group: Option<String>,

100 +}

101 +

102 +/// A region delimiter.

103 +#[derive(Debug)]

104 +pub(super) enum Boundary {

105 + /// Literal alternatives, longest first.

106 + Literals(Vec<String>),

107 + Pattern(Box<StreamingPattern>),

108 +}

109 +

110 +/// One boundary the executor watches for in a given state.

111 +#[derive(Debug, Clone, Copy, PartialEq, Eq)]

112 +pub(super) struct Watch {

113 + pub region: usize,

114 + pub kind: WatchKind,

115 +}

116 +

117 +#[derive(Debug, Clone, Copy, PartialEq, Eq)]

118 +pub(super) enum WatchKind {

119 + Open,

120 + Close,

121 +}

122 +

123 +/// The boundaries watched in one parser mode.

124 +#[derive(Debug, Default)]

125 +pub(super) struct WatchSet {

126 + pub boundaries: Vec<Watch>,

127 + /// Literals, deduplicated, whose occurrences are the only positions any of

128 + /// the boundaries can match at.

129 + pub markers: Vec<String>,

130 +}

131 +

132 +impl ResponseTemplate {

133 + /// Compile the `response_template` object from `tokenizer_config.json`.

134 + pub fn from_json(value: &Value) -> Result<Self> {

135 + let spec = TemplateSpec::from_value(value)?;

136 +

137 + let regions = spec

138 + .fields

139 + .iter()

140 + .enumerate()

141 + .map(|(index, (name, field))| Region::compile(index, name, field))

142 + .collect::<Result<Vec<_>>>()?;

143 +

144 + // A field without an open anchor is the implicit-open / leftover sink; at most one is allowed.

145 + let implicit_fields: Vec<_> =

146 + regions.iter().enumerate().filter(|(_, region)| region.open.is_none()).collect();

147 + if implicit_fields.len() > 1 {

148 + let names: Vec<_> =

149 + implicit_fields.iter().map(|(_, region)| region.name.as_str()).collect();

150 + return Err(invalid!(

151 + "At most one field may omit 'open'/'open_pattern' (that field becomes the \

152 + implicit-open / leftover sink). Found: {}",

153 + names.join(", ")

154 + ));

155 + }

156 + let implicit = implicit_fields.first().map(|(index, _)| *index);

157 +

158 + let start_anchor = match spec.start_anchor()? {

159 + AnchorSpec::Literals(mut literals) => {

160 + // Sort longest-first so alternation prefers the longer alternative when both could match.

161 + literals.sort_by_key(|literal| std::cmp::Reverse(literal.len()));

162 + let alternation: Vec<_> =

163 + literals.iter().map(|literal| regex_syntax::escape(literal)).collect();

164 + compile_regex("response_template", &alternation.join("|"))?

165 + }

166 + AnchorSpec::Pattern(pattern) => compile_regex("response_template", pattern)?,

167 + };

168 +

169 + let opens = regions.iter().enumerate().filter_map(|(region, spec)| {

170 + let watch = Watch {

171 + region,

172 + kind: WatchKind::Open,

173 + };

174 + Some((watch, spec.open.as_ref()?))

175 + });

176 + let implicit_close = implicit.and_then(|region| {

177 + let watch = Watch {

178 + region,

179 + kind: WatchKind::Close,

180 + };

181 + Some((watch, regions[region].close.as_ref()?))

182 + });

183 + let idle_watch = WatchSet::new(opens.chain(implicit_close));

184 +

185 + Ok(Self {

186 + start_anchor,

187 + regions,

188 + implicit,

189 + idle_watch,

190 + warned_missing_anchor: AtomicBool::new(false),

191 + })

192 + }

193 +

194 + /// Return the start and end offsets of the last start-anchor match in

195 + /// `prompt`.

196 + pub(super) fn last_start_anchor(&self, prompt: &str) -> Option<(usize, usize)> {

197 + self.start_anchor

198 + .find_iter(prompt)

199 + .last()

200 + .map(|found| (found.start(), found.end()))

201 + }

202 +

203 + /// Log once per template that the prompt carried no start anchor.

204 + pub(super) fn warn_missing_anchor_once(&self) {

205 + if !self.warned_missing_anchor.swap(true, Ordering::Relaxed) {

206 + tracing::warn!(

207 + "response_template start anchor not found in the prompt; parsing starts from the \

208 + template's initial state (the chat template may not match the checkpoint's \

209 + response_template)"

210 + );

211 + }

212 + }

213 +

214 + /// The boundary of a watch entry.

215 + pub(super) fn boundary(&self, watch: Watch) -> &Boundary {

216 + let region = &self.regions[watch.region];

217 + match watch.kind {

218 + WatchKind::Open => region.open.as_ref(),

219 + WatchKind::Close => region.close.as_ref(),

220 + }

221 + .expect("watched boundaries exist")

222 + }

223 +}

224 +

225 +impl Region {

226 + /// Validate the spec of the field at `index` and compile it.

227 + fn compile(index: usize, name: &str, field: &FieldSpec) -> Result<Self> {

228 + let scope = format!("Field '{name}'");

229 + let open = field.open(&scope)?.map(|spec| Boundary::compile(&scope, spec)).transpose()?;

230 + let close = field.close(&scope)?.map(|spec| Boundary::compile(&scope, spec)).transpose()?;

231 +

232 + let content = ContentParser::new(&scope, field.content, &field.content_args)?;

233 + let repeat = match (field.repeats, &field.join) {

234 + (false, None) => Repeat::Once,

235 + (true, None) => Repeat::Each,

236 + (true, Some(join)) => Repeat::Join(join.clone()),

237 + (false, Some(_)) => return Err(invalid!("{scope}: 'join' requires 'repeats': true")),

238 + };

239 + let transform = match (&field.transform, field.transform_each) {

240 + (Some(template), each) => Some(FieldTransform {

241 + template: Transform::compile(&scope, template)?,

242 + each,

243 + }),

244 + (None, false) => None,

245 + (None, true) => {

246 + return Err(invalid!(

247 + "{scope}: transform_each is set but no transform was provided"

248 + ));

249 + }

250 + };

251 +

252 + let captured: Vec<&str> =

253 + [&open, &close].into_iter().flatten().flat_map(Boundary::group_names).collect();

254 + match &transform {

255 + None if !captured.is_empty() => {

256 + // Named captures only reach the output through a transform, so flag any that would be silently dropped.

257 + return Err(invalid!(

258 + "{scope}: open_pattern/close_pattern declares named group(s) {captured:?}, but \

259 + the field has no 'transform'. Named captures are only surfaced through a \

260 + 'transform' template (where they appear alongside 'content'). Either add a \

261 + 'transform' that uses the captures, or remove the named groups from the pattern."

262 + ));

263 + }

264 + // Without `transform_each`, the scope is the opener's captures plus `content`;

265 + // any other placeholder fails on every match.

266 + Some(FieldTransform {

267 + template,

268 + each: false,

269 + }) => {

270 + let open_groups: Vec<&str> = open.iter().flat_map(Boundary::group_names).collect();

271 + if let Some(root) = template

272 + .placeholder_roots()

273 + .find(|root| *root != "content" && !open_groups.contains(root))

274 + {

275 + return Err(invalid!(

276 + "{scope}: transform placeholder '{{{root}}}' is neither 'content' nor a \

277 + named group of open_pattern"

278 + ));

279 + }

280 + }

281 + _ => {}

282 + }

283 +

284 + let name = FieldName::parse(&scope, name)?;

285 + let kind = match name.text_role() {

286 + Some(role) => {

287 + let strip = match (content, &transform) {

288 + (ContentParser::Text(TextArgs { strip }), None) => strip,

289 + _ => {

290 + return Err(unsupported!(

291 + "{scope}: text and reasoning fields must use the 'text' content \

292 + parser without a transform"

293 + ));

294 + }

295 + };

296 + RegionKind::Text(TextRegion {

297 + role,

298 + strip,

299 + repeat,

300 + })

301 + }

302 + None => {

303 + if matches!(repeat, Repeat::Join(_)) {

304 + return Err(unsupported!(

305 + "{scope}: 'join' requires each match to parse to a string"

306 + ));

307 + }

308 + let name_group = name_group(transform.as_ref(), open.as_ref());

309 + RegionKind::ToolCalls(ToolCallRegion {

310 + content,

311 + transform,

312 + name_group,

313 + })

314 + }

315 + };

316 +

317 + let close_watch = WatchSet::new(close.as_ref().map(|close| {

318 + let watch = Watch {

319 + region: index,

320 + kind: WatchKind::Close,

321 + };

322 + (watch, close)

323 + }));

324 +

325 + Ok(Self {

326 + name,

327 + open,

328 + close,

329 + close_watch,

330 + optional: field.optional,

331 + kind,

332 + })

333 + }

334 +}

335 +

336 +/// The opener capture group that a non-`transform_each` transform uses as the

337 +/// function name, if any.

338 +fn name_group(transform: Option<&FieldTransform>, open: Option<&Boundary>) -> Option<String> {

339 + let (

340 + FieldTransform {

341 + template,

342 + each: false,

343 + },

344 + open,

345 + ) = (transform?, open?)

346 + else {

347 + return None;

348 + };

349 + let path = template

350 + .placeholder_at(&["function", "name"])

351 + .or_else(|| template.placeholder_at(&["name"]))?;

352 + match path {

353 + [root] if open.group_names().any(|group| group == root) => Some(root.clone()),

354 + _ => None,

355 + }

356 +}

357 +

358 +impl FieldName {

359 + /// Map a field name to a reportable field.

360 + fn parse(scope: &str, name: &str) -> Result<Self> {

361 + Ok(match name {

362 + "content" => Self::Content,

363 + "reasoning_content" => Self::ReasoningContent,

364 + "thinking" => Self::Thinking,

365 + "tool_calls" => Self::ToolCalls,

366 + _ => {

367 + return Err(unsupported!(

368 + "{scope}: only 'content', 'reasoning_content'/'thinking', and 'tool_calls' \

369 + fields can be reported"

370 + ));

371 + }

372 + })

373 + }

374 +

375 + /// The role of a text field; `None` for `tool_calls`.

376 + fn text_role(self) -> Option<TextRole> {

377 + match self {

378 + Self::Content => Some(TextRole::Content),

379 + Self::ReasoningContent | Self::Thinking => Some(TextRole::Reasoning),

380 + Self::ToolCalls => None,

381 + }

382 + }

383 +

384 + pub fn as_str(self) -> &'static str {

385 + match self {

386 + Self::Content => "content",

387 + Self::ReasoningContent => "reasoning_content",

388 + Self::Thinking => "thinking",

389 + Self::ToolCalls => "tool_calls",

390 + }

391 + }

392 +}

393 +

394 +impl fmt::Display for FieldName {

395 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {

396 + f.write_str(self.as_str())

397 + }

398 +}

399 +

400 +impl Boundary {

401 + /// Compile an anchor into a boundary.

402 + fn compile(scope: &str, spec: AnchorSpec<'_>) -> Result<Self> {

403 + Ok(match spec {

404 + AnchorSpec::Literals(mut literals) => {

405 + // Sort longest-first so the longer alternative wins when both match.

406 + literals.sort_by_key(|literal| std::cmp::Reverse(literal.len()));

407 + Self::Literals(literals)

408 + }

409 + AnchorSpec::Pattern(pattern) => {

410 + Self::Pattern(Box::new(StreamingPattern::new(scope, pattern)?))

411 + }

412 + })

413 + }

414 +

415 + /// Literals whose occurrences are the only positions this boundary can match at.

416 + pub fn markers(&self) -> &[String] {

417 + match self {

418 + Self::Literals(literals) => literals,

419 + Self::Pattern(pattern) => pattern.prefixes(),

420 + }

421 + }

422 +

423 + /// Named capture groups.

424 + pub fn group_names(&self) -> impl Iterator<Item = &str> {

425 + match self {

426 + Self::Literals(_) => None,

427 + Self::Pattern(pattern) => Some(pattern.group_names()),

428 + }

429 + .into_iter()

430 + .flatten()

431 + }

432 +

433 + /// Resolve this boundary at the start of `text`.

434 + pub fn resolve(&self, text: &str, eof: bool) -> Resolution {

435 + let markers = self.markers();

436 + // A marker the available text is a proper prefix of: more input decides.

437 + let partial = !eof

438 + && markers

439 + .iter()

440 + .any(|marker| marker.len() > text.len() && marker.starts_with(text));

441 + match self {

442 + Self::Literals(literals) => {

443 + if partial {

444 + return Resolution::Pending;

445 + }

446 + match literals.iter().find(|literal| text.starts_with(literal.as_str())) {

447 + Some(literal) => Resolution::Matched {

448 + len: literal.len(),

449 + captures: Captures::new(),

450 + },

451 + None => Resolution::NoMatch,

452 + }

453 + }

454 + Self::Pattern(pattern) => {

455 + if markers.iter().any(|marker| text.starts_with(marker.as_str())) {

456 + pattern.resolve(text, eof)

457 + } else if partial {

458 + Resolution::Pending

459 + } else {

460 + Resolution::NoMatch

461 + }

462 + }

463 + }

464 + }

465 +}

466 +

467 +impl WatchSet {

468 + /// Collect the watched boundaries and their deduplicated markers.

469 + fn new<'a>(entries: impl IntoIterator<Item = (Watch, &'a Boundary)>) -> Self {

470 + let mut set = Self::default();

471 + for (watch, boundary) in entries {

472 + set.boundaries.push(watch);

473 + for marker in boundary.markers() {

474 + if !set.markers.contains(marker) {

475 + set.markers.push(marker.clone());

476 + }

477 + }

478 + }

479 + set

480 + }

481 +}

rust/src/parser/src/unified/hf/tests.rs new

+933

@@ -0,0 +1,933 @@

1 +// SPDX-License-Identifier: Apache-2.0

2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +//! Executor tests.

5 +//!

6 +//! Template fixtures and expected values are ported from Transformers

7 +//! `tests/utils/test_chat_parsing.py` at 6d43ab4008, with the expected output

8 +//! dict re-expressed as aggregated unified parser events (`thinking` becomes

9 +//! `reasoning`).

10 +

11 +use std::collections::BTreeMap;

12 +use std::sync::Arc;

13 +

14 +use serde_json::{Value, json};

15 +use vllm_tokenizer::test_utils::TestTokenizer;

16 +use vllm_tokenizer::{DecodedText, TokenAnchor, TokenAttribution, Tokenizer as _};

17 +

18 +use super::{HfTemplateError, HfUnifiedParser, ResponseTemplate};

19 +use crate::tool::Tool;

20 +use crate::tool::test_utils::split_by_chars;

21 +use crate::unified::{UnifiedParser, UnifiedParserError, UnifiedParserEvent, UnifiedParserOutput};

22 +

23 +fn cohere_template() -> Value {

24 + json!({

25 + "defaults": {"role": "assistant"},

26 + "start_anchor": "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>",

27 + "fields": {

28 + "content": {"open": "<|START_RESPONSE|>", "close": "<|END_RESPONSE|>", "content": "text"},

29 + "thinking": {"open": "<|START_THINKING|>", "close": "<|END_THINKING|>", "content": "text"},

30 + "tool_calls": {

31 + "open": "<|START_ACTION|>",

32 + "close": "<|END_ACTION|>",

33 + "content": "json",

34 + "transform_each": true,

35 + "transform": {"type": "function", "function": {"name": "{tool_name}", "arguments": "{parameters}"}},

36 + },

37 + },

38 + })

39 +}

40 +

41 +fn gpt_oss_template() -> Value {

42 + json!({

43 + "defaults": {"role": "assistant"},

44 + "start_anchor": "<|start|>assistant",

45 + "fields": {

46 + "thinking": {"open": "<|channel|>analysis<|message|>", "close": "<|end|>", "content": "text"},

47 + "content": {"open": "<|channel|>final<|message|>", "close": "<|end|>", "content": "text"},

48 + "tool_calls": {

49 + "open_pattern": r"<\|channel\|>commentary to=functions\.(?P<name>\w+).*?<\|message\|>",

50 + "close": "<|call|>",

51 + "repeats": true,

52 + "content": "json",

53 + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},

54 + },

55 + },

56 + })

57 +}

58 +

59 +fn smollm_template() -> Value {

60 + json!({

61 + "defaults": {"role": "assistant"},

62 + "start_anchor": "<|im_start|>assistant\n",

63 + "fields": {

64 + "thinking": {"open": "<think>", "close": "</think>", "content": "text"},

65 + "tool_calls": {

66 + "open": "<tool_call>",

67 + "close": "</tool_call>",

68 + "repeats": true,

69 + "content": "json",

70 + "transform": {"type": "function", "function": "{content}"},

71 + },

72 + "content": {"close": "<|im_end|>", "content": "text"},

73 + },

74 + })

75 +}

76 +

77 +fn qwen3_template() -> Value {

78 + json!({

79 + "defaults": {"role": "assistant"},

80 + "start_anchor": "<|im_start|>assistant\n",

81 + "fields": {

82 + "thinking": {"open": "<think>", "close": "</think>", "content": "text"},

83 + "tool_calls": {

84 + "open_pattern": r"<tool_call>\s*<function=(?P<name>\w+)>",

85 + "close": "</tool_call>",

86 + "repeats": true,

87 + "content": "xml-inline",

88 + "content_args": {

89 + "tag_pattern": r"<parameter=(?P<key>\w+)>\s*(?P<value>.*?)\s*</parameter>",

90 + "value_parser": {"name": "json", "args": {"allow_non_json": true}},

91 + },

92 + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},

93 + },

94 + },

95 + })

96 +}

97 +

98 +/// Identical to the template shipped in the Gemma 4 IT checkpoints.

99 +fn gemma4_template() -> Value {

100 + json!({

101 + "defaults": {"role": "assistant"},

102 + "start_anchor": ["<|turn>model\n", "<tool_response|>"],

103 + "fields": {

104 + "thinking": {"open": "<|channel>thought\n", "close": "<channel|>", "content": "text"},

105 + "tool_calls": {

106 + "open_pattern": r"<\|tool_call>call:(?P<name>\w+)",

107 + "close": "<tool_call|>",

108 + "repeats": true,

109 + "content": "json",

110 + "content_args": {"unquoted_keys": true, "string_delims": [["<|\"|>", "<|\"|>"]]},

111 + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},

112 + },

113 + "content": {"close": ["<turn|>", "<|tool_response>", "<eos>"], "content": "text"},

114 + },

115 + })

116 +}

117 +

118 +/// A routed-message protocol with XML-style invocations: no implicit field

119 +/// (text outside regions is discarded), Unicode word boundaries and lazy

120 +/// quantifiers in the opener, `xml-inline` arguments, and joined content.

121 +fn invoke_template() -> Value {

122 + json!({

123 + "defaults": {"role": "assistant"},

124 + "start_anchor": "<|begin|>assistant",

125 + "fields": {

126 + "reasoning_content": {"open_pattern": r"to=self<\|msg\|>", "close": "<|pause|>", "content": "text"},

127 + "tool_calls": {

128 + "open_pattern": r#"<invoke\b[^>]*?\bname="(?P<name>[^"]+)">"#,

129 + "close": "</invoke>",

130 + "repeats": true,

131 + "content": "xml-inline",

132 + "content_args": {

133 + "tag_pattern": r#"<param\b[^>]*?\bname="(?P<key>[^"]+)"[^>]*?>(?P<value>.*?)</param>"#,

134 + "value_parser": {"name": "json", "args": {"allow_non_json": true}},

135 + },

136 + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},

137 + },

138 + "content": {

139 + "open_pattern": r"to=(?:user|note)<\|msg\|>",

140 + "close": ["<|end|>", "<|pause|>"],

141 + "repeats": true,

142 + "join": "",

143 + "content": "text",

144 + },

145 + },

146 + })

147 +}

148 +

149 +fn compile(template: Value) -> Arc<ResponseTemplate> {

150 + Arc::new(ResponseTemplate::from_json(&template).unwrap())

151 +}

152 +

153 +fn tokenizer() -> Arc<TestTokenizer> {

154 + Arc::new(TestTokenizer::new())

155 +}

156 +

157 +/// Parse `chunks` after initializing from `prompt`, returning all events.

158 +fn parse_events(

159 + template: &Arc<ResponseTemplate>,

160 + tools: &[Tool],

161 + prompt: &str,

162 + chunks: &[&str],

163 +) -> Result<Vec<UnifiedParserEvent>, UnifiedParserError> {

164 + let tokenizer = tokenizer();

165 + let prompt_ids = tokenizer.encode(prompt, false).unwrap();

166 + let mut parser = HfUnifiedParser::new(template.clone(), tools, tokenizer);

167 + parser.initialize(&prompt_ids)?;

168 + let mut output = UnifiedParserOutput::default();

169 + for chunk in chunks {

170 + parser.parse_into(DecodedText::unattributed(*chunk), &mut output)?;

171 + }

172 + output.append(parser.finish()?);

173 + Ok(output.events)

174 +}

175 +

176 +/// Aggregate events into the shape of the Transformers output dict.

177 +fn message(events: &[UnifiedParserEvent]) -> Value {

178 + let mut reasoning = String::new();

179 + let mut content = String::new();

180 + let mut calls: BTreeMap<usize, (String, String)> = BTreeMap::new();

181 + for event in events {

182 + match event {

183 + UnifiedParserEvent::Text(text) => content.push_str(text),

184 + UnifiedParserEvent::Reasoning(piece) => reasoning.push_str(&piece.text),

185 + UnifiedParserEvent::ToolCall(delta) => {

186 + let call = calls.entry(delta.tool_index).or_default();

187 + if let Some(name) = &delta.name {

188 + call.0.clone_from(name);

189 + }

190 + call.1.push_str(&delta.arguments);

191 + }

192 + }

193 + }

194 + let mut out = serde_json::Map::new();

195 + if !reasoning.is_empty() {

196 + out.insert("reasoning".to_string(), reasoning.into());

197 + }

198 + if !content.is_empty() {

199 + out.insert("content".to_string(), content.into());

200 + }

201 + if !calls.is_empty() {

202 + let calls = calls

203 + .into_values()

204 + .map(|(name, arguments)| {

205 + let arguments =

206 + serde_json::from_str(&arguments).unwrap_or(Value::String(arguments));

207 + json!({"name": name, "arguments": arguments})

208 + })

209 + .collect();

210 + out.insert("tool_calls".to_string(), Value::Array(calls));

211 + }

212 + Value::Object(out)

213 +}

214 +

215 +fn parse(template: Value, text: &str) -> Value {

216 + message(&parse_events(&compile(template), &[], "", &[text]).unwrap())

217 +}

218 +

219 +/// Split `text` at pseudo-random char boundaries (xorshift, seeded).

220 +fn random_chunks(text: &str, seed: u64) -> Vec<&str> {

221 + let mut state = seed | 1;

222 + let mut next = move || {

223 + state ^= state << 13;

224 + state ^= state >> 7;

225 + state ^= state << 17;

226 + state

227 + };

228 + let mut chunks = Vec::new();

229 + let mut start = 0;

230 + for (index, _) in text.char_indices().skip(1) {

231 + if next() % 3 == 0 {

232 + chunks.push(&text[start..index]);

233 + start = index;

234 + }

235 + }

236 + chunks.push(&text[start..]);

237 + chunks

238 +}

239 +

240 +#[test]

241 +fn upstream_fixtures_match_transformers_values() {

242 + let cases = [

243 + (

244 + cohere_template(),

245 + "<|START_THINKING|>I should call a tool.<|END_THINKING|><|START_ACTION|>[\n {\"tool_call_id\": \"0\", \"tool_name\": \"simple_tool\", \"parameters\": {\"temperature_format\": \"Celsius\"}}\n]<|END_ACTION|><|END_OF_TURN_TOKEN|>",

246 + json!({

247 + "reasoning": "I should call a tool.",

248 + "tool_calls": [{"name": "simple_tool", "arguments": {"temperature_format": "Celsius"}}],

249 + }),

250 + ),

251 + (

252 + gpt_oss_template(),

253 + "<|channel|>analysis<|message|>We will call function get_current_weather.<|end|><|start|>assistant<|channel|>commentary to=functions.get_current_weather <|constrain|>json<|message|>{\n \"location\": \"San Francisco, CA\"\n}",

254 + json!({

255 + "reasoning": "We will call function get_current_weather.",

256 + "tool_calls": [{"name": "get_current_weather", "arguments": {"location": "San Francisco, CA"}}],

257 + }),

258 + ),

259 + (

260 + gpt_oss_template(),

262 + json!({

263 + "reasoning": "User asks a simple math question: 2+2 = 4. Provide answer.",

264 + "content": "2",

265 + }),

266 + ),

267 + (

268 + smollm_template(),

269 + "<think>\nOkay, let me greet them.\n</think>\n\n<tool_call>{\"name\": \"greet_user\", \"arguments\": {\"greeting\": \"Hello!\"}}</tool_call>",

270 + json!({

271 + "reasoning": "Okay, let me greet them.",

272 + "tool_calls": [{"name": "greet_user", "arguments": {"greeting": "Hello!"}}],

273 + }),

274 + ),

275 + (

276 + smollm_template(),

277 + "<tool_call>{\"name\": \"get_weather\", \"arguments\": {\"city\": \"Paris\"}}</tool_call>",

278 + json!({"tool_calls": [{"name": "get_weather", "arguments": {"city": "Paris"}}]}),

279 + ),

280 + (

281 + smollm_template(),

282 + "<think>\nOkay, gravity.</think>\nSome content about gravity goes here!",

283 + json!({"reasoning": "Okay, gravity.", "content": "Some content about gravity goes here!"}),

284 + ),

285 + (

286 + qwen3_template(),

287 + "<tool_call>\n<function=get_weather>\n<parameter=locations>\n[{\"country\": \"France\", \"city\": \"Paris\"}]\n</parameter>\n<parameter=temp_units>\ncelsius\n</parameter>\n</function>\n</tool_call>",

288 + json!({

289 + "tool_calls": [{

290 + "name": "get_weather",

291 + "arguments": {"locations": [{"country": "France", "city": "Paris"}], "temp_units": "celsius"},

292 + }],

293 + }),

294 + ),

295 + (

296 + gemma4_template(),

297 + "<|channel>thought\nThe user is asking for the temperature.<channel|><|tool_call>call:get_current_temperature{detail_level:0,location:<|\"|>Paris, France<|\"|>,unit:<|\"|>celsius<|\"|>}<tool_call|><|tool_response>",

298 + json!({

299 + "reasoning": "The user is asking for the temperature.",

300 + "tool_calls": [{

301 + "name": "get_current_temperature",

302 + "arguments": {"detail_level": 0, "location": "Paris, France", "unit": "celsius"},

303 + }],

304 + }),

305 + ),

306 + (

307 + gemma4_template(),

308 + "<|channel>thought\nLet me call the tool.<channel|><|tool_call>call:foo{bool_value:true,list_value:[<|\"|>foo<|\"|>,<|\"|>bar<|\"|>],null_value:null,number_value:1,string_value:<|\"|>foo<|\"|>,struct_value:{foo:<|\"|>bar<|\"|>}}<tool_call|>",

309 + json!({

310 + "reasoning": "Let me call the tool.",

311 + "tool_calls": [{

312 + "name": "foo",

313 + "arguments": {

314 + "bool_value": true,

315 + "list_value": ["foo", "bar"],

316 + "null_value": null,

317 + "number_value": 1,

318 + "string_value": "foo",

319 + "struct_value": {"foo": "bar"},

320 + },

321 + }],

322 + }),

323 + ),

324 + ];

325 + for (template, text, expected) in cases {

326 + assert_eq!(parse(template, text), expected, "{text}");

327 + }

328 +}

329 +

330 +/// Upstream fixtures whose patterns have no literal prefix are rejected at load.

331 +#[test]

332 +fn patterns_without_literal_prefix_are_unsupported() {

333 + let ernie = json!({

334 + "start_anchor": "Assistant:",

335 + "fields": {"thinking": {"open_pattern": r"(?:^|<think>\s*)", "close": "</think>"}},

336 + });

337 + let inkling = json!({

338 + "start_anchor": "<|message_model|>",

339 + "fields": {"content": {"open_pattern": r"(?:<\|message_model\|>)?[^<]*<\|content_text\|>", "close": "<|end_message|>"}},

340 + });

341 + for template in [ernie, inkling] {

342 + assert!(matches!(

343 + ResponseTemplate::from_json(&template),

344 + Err(HfTemplateError::Unsupported { .. })

345 + ));

346 + }

347 +}

348 +

349 +#[test]

350 +fn streaming_matches_whole_string_for_every_chunking() {

351 + let fixtures = [

352 + (

353 + cohere_template(),

354 + "<|START_THINKING|>I should call a tool.<|END_THINKING|><|START_ACTION|>[{\"tool_call_id\": \"0\", \"tool_name\": \"simple_tool\", \"parameters\": {\"a\": 1}}]<|END_ACTION|>",

355 + ),

356 + (

357 + gpt_oss_template(),

359 + ),

360 + (

361 + gpt_oss_template(),

362 + "<|channel|>analysis<|message|>Let me check.<|end|><|start|>assistant<|channel|>commentary to=functions.get_current_weather <|constrain|>json<|message|>{\"location\": \"San Francisco, CA\"}<|call|>",

363 + ),

364 + (

365 + smollm_template(),

366 + "<think>thinking</think>\n<tool_call>{\"name\": \"fn\", \"arguments\": {\"x\": 1}}</tool_call>",

367 + ),

368 + (

369 + qwen3_template(),

370 + "<think>short thought</think>\n<tool_call>\n<function=get_weather>\n<parameter=city>\nParis\n</parameter>\n</function>\n</tool_call>",

371 + ),

372 + (

373 + gemma4_template(),

374 + "<|channel>thought\nhi 天气<channel|>前言 <|tool_call>call:foo{a:1,b:<|\"|>bar<|\"|>}<tool_call|> after",

375 + ),

376 + (

377 + invoke_template(),

379 + ),

380 + ];

381 + for (template, text) in fixtures {

382 + let template = compile(template);

383 + let expected = message(&parse_events(&template, &[], "", &[text]).unwrap());

384 + let mut chunkings: Vec<Vec<&str>> =

385 + [1, 2, 3, 5, 7, 13, 31].map(|step| split_by_chars(text, step)).into();

386 + chunkings.extend((0..30).map(|seed| random_chunks(text, 0xC0DE_5EED + seed)));

387 + for chunks in chunkings {

388 + assert_eq!(chunks.concat(), text);

389 + let actual = message(&parse_events(&template, &[], "", &chunks).unwrap());

390 + assert_eq!(actual, expected, "{chunks:?}");

391 + }

392 + }

393 +}

394 +

395 +#[test]

396 +fn gemma4_streams_text_and_starts_calls_at_the_opener() {

397 + let template = compile(gemma4_template());

398 + let chunks = [

399 + "<|channel>thought\n",

400 + "Check ",

401 + "the weather. ",

402 + "<channel|>Sure",

403 + ". <|tool_call>call:get_weather",

404 + "{location:<|\"|>Paris<|\"|>}",

405 + "<tool_call|>",

406 + ];

407 + let tokenizer = tokenizer();

408 + let mut parser = HfUnifiedParser::new(template, &[], tokenizer);

409 + parser.initialize(&[]).unwrap();

410 + let steps: Vec<_> = chunks

411 + .iter()

412 + .map(|chunk| {

413 + let mut output = UnifiedParserOutput::default();

414 + parser.parse_into(DecodedText::unattributed(*chunk), &mut output).unwrap();

415 + (*chunk, output.events)

416 + })

417 + .collect();

418 + expect_test::expect![[r#"

419 + [

420 + (

421 + "<|channel>thought\n",

422 + [],

423 + ),

424 + (

425 + "Check ",

426 + [

427 + Reasoning(

428 + DecodedText {

429 + text: "Check",

430 + attributions: [],

431 + },

432 + ),

433 + ],

434 + ),

435 + (

436 + "the weather. ",

437 + [

438 + Reasoning(

439 + DecodedText {

440 + text: " the weather.",

441 + attributions: [],

442 + },

443 + ),

444 + ],

445 + ),

446 + (

447 + "<channel|>Sure",

448 + [

449 + Text(

450 + "Sure",

451 + ),

452 + ],

453 + ),

454 + (

455 + ". <|tool_call>call:get_weather",

456 + [

457 + Text(

458 + ".",

459 + ),

460 + ],

461 + ),

462 + (

463 + "{location:<|\"|>Paris<|\"|>}",

464 + [

465 + ToolCall(

466 + ToolCallDelta {

467 + tool_index: 0,

468 + name: Some(

469 + "get_weather",

470 + ),

471 + arguments: "",

472 + },

473 + ),

474 + ],

475 + ),

476 + (

477 + "<tool_call|>",

478 + [

479 + ToolCall(

480 + ToolCallDelta {

481 + tool_index: 0,

482 + name: None,

483 + arguments: "{\"location\":\"Paris\"}",

484 + },

485 + ),

486 + ],

487 + ),

488 + ]

489 + "#]]

490 + .assert_debug_eq(&steps);

491 +}

492 +

493 +#[test]

494 +fn invoke_protocol_discards_unrouted_text_and_joins_content() {

496 + expect_test::expect![[r#"

497 + Object {

498 + "reasoning": String("plan"),

499 + "content": String("Hello,world"),

500 + "tool_calls": Array [

501 + Object {

502 + "name": String("get"),

503 + "arguments": Object {

504 + "n": String("x"),

505 + },

506 + },

507 + ],

508 + }

509 + "#]]

510 + .assert_debug_eq(&parse(invoke_template(), text));

511 +

512 + // Unicode word boundaries: `é` is a word character, so `éname=` is not the `name` attribute.

513 + let text = "<invoke title=\"天气\" name=\"获取\"><param name=\"城市\">北京</param></invoke><invoke éname=\"x\">";

514 + assert_eq!(

515 + parse(invoke_template(), text),

516 + json!({"tool_calls": [{"name": "获取", "arguments": {"城市": "北京"}}]})

517 + );

518 +}

519 +

520 +#[test]

521 +fn join_concatenates_repeated_matches() {

522 + let template = json!({

523 + "defaults": {"role": "assistant"},

524 + "start_anchor": "<|assistant|>",

525 + "fields": {

526 + "thinking": {"open": "<think>", "close": "</think>", "repeats": true, "join": " "},

527 + "content": {"repeats": true, "join": " "},

528 + },

529 + });

530 + assert_eq!(

531 + parse(

532 + template.clone(),

533 + "<think>first</think>middle<think>second</think>done"

534 + ),

535 + json!({"reasoning": "first second", "content": "middle done"})

536 + );

537 + assert_eq!(

538 + parse(template, "<think>only</think>"),

539 + json!({"reasoning": "only"})

540 + );

541 +}

542 +

543 +#[test]

544 +fn transforms_with_dotted_paths() {

545 + let template = json!({

546 + "start_anchor": "<|assistant|>",

547 + "fields": {

548 + "tool_calls": {

549 + "open": "<actions>",

550 + "close": "</actions>",

551 + "content": "json",

552 + "transform_each": true,

553 + "transform": {"type": "function", "function": {"name": "{fn.name}", "arguments": "{fn.args}"}},

554 + },

555 + },

556 + });

557 + assert_eq!(

558 + parse(

559 + template,

560 + r#"<actions>[{"fn": {"name": "a", "args": {"x": 1}}}, {"fn": {"name": "b", "args": {}}}]</actions>"#

561 + ),

562 + json!({"tool_calls": [{"name": "a", "arguments": {"x": 1}}, {"name": "b", "arguments": {}}]})

563 + );

564 +}

565 +

566 +#[test]

567 +fn literal_boundaries() {

568 + let template = json!({

569 + "start_anchor": "<|assistant|>",

570 + "fields": {"content": {"open": ["<a>", "<bb>"], "close": ["</a>", "</bb>"]}},

571 + });

572 + for (opener, closer) in [("<a>", "</a>"), ("<bb>", "</bb>"), ("<a>", "</bb>")] {

573 + assert_eq!(

574 + parse(template.clone(), &format!("{opener}hi{closer}")),

575 + json!({"content": "hi"})

576 + );

577 + }

578 +

579 + // A literal that is a strict prefix of another in the same list defers at the buffer edge.

580 + let template = compile(json!({

581 + "start_anchor": "<|assistant|>",

582 + "fields": {"content": {"open": "<x>", "close": ["END", "ENDX"]}},

583 + }));

584 + let events = parse_events(&template, &[], "", &["<x>hiEND", " more"]).unwrap();

585 + assert_eq!(message(&events), json!({"content": "hi"}));

586 +

587 + // A field without a close runs to the end of the stream.

588 + let template =

589 + json!({"start_anchor": "<|assistant|>", "fields": {"content": {"open": "<resp>"}}});

590 + assert_eq!(

591 + parse(template, "<resp>hello world"),

592 + json!({"content": "hello world"})

593 + );

594 +}

595 +

596 +#[test]

597 +fn required_field_missing_fails_at_finish() {

598 + let template = compile(json!({

599 + "start_anchor": "<|assistant|>",

600 + "fields": {"content": {"open": "<response>", "close": "</response>", "optional": false}},

601 + }));

602 + let error = parse_events(&template, &[], "", &["no response here"]).unwrap_err();

603 + expect_test::expect![[r#"

604 + ParsingFailed {

605 + message: "Required response_template fields missing from parsed output: [\"content\"]",

606 + }

607 + "#]]

608 + .assert_debug_eq(&error);

609 +}

610 +

611 +#[test]

612 +fn prefix_sets_the_initial_state() {

613 + let template = compile(qwen3_template());

614 +

615 + // The prompt ends inside the thinking block; the prefilled newline is stripped.

617 + let events = parse_events(&template, &[], prompt, &["Let me think...</think>"]).unwrap();

618 + assert_eq!(message(&events), json!({"reasoning": "Let me think..."}));

619 +

620 + // Only the slice after the last anchor matters.

622 + let events = parse_events(&template, &[], prompt, &["done</think>"]).unwrap();

623 + assert_eq!(message(&events), json!({"reasoning": "done"}));

624 +

625 + // A prefix ending mid-delimiter is completed by the generated text.

626 + let events = parse_events(

627 + &template,

628 + &[],

629 + "<|im_start|>assistant\n<thi",

630 + &["nk>real body</think>"],

631 + )

632 + .unwrap();

633 + assert_eq!(message(&events), json!({"reasoning": "real body"}));

634 +

635 + // Gemma 4 continues inside the same turn after a tool response.

636 + let template = compile(gemma4_template());

637 + let prompt = "<|turn>model\n<|tool_call>call:f{}<tool_call|><|tool_response>response:f{}<tool_response|><|channel>thought\n";

638 + let events = parse_events(&template, &[], prompt, &["Done.<channel|>It is sunny."]).unwrap();

639 + assert_eq!(

640 + message(&events),

641 + json!({"reasoning": "Done.", "content": "It is sunny."})

642 + );

643 +

644 + // Prompt text is never emitted, but it counts for leading-whitespace stripping.

645 + let events = parse_events(&template, &[], "<|turn>model\nSure,", &[" here", " it is"]).unwrap();

646 + assert_eq!(message(&events), json!({"content": " here it is"}));

647 +}

648 +

649 +/// Diverges from Transformers, which runs the whole prompt through the parser

650 +/// when the start anchor is absent.

651 +#[test]

652 +fn missing_start_anchor_starts_from_the_initial_state() {

653 + let template = compile(smollm_template());

654 + let events = parse_events(&template, &[], "<think>\n", &["hi</think>"]).unwrap();

655 + assert_eq!(message(&events), json!({"content": "hi</think>"}));

656 +}

657 +

658 +#[test]

659 +fn tools_convert_string_arguments() {

660 + let template = compile(json!({

661 + "start_anchor": "<|im_start|>assistant\n",

662 + "fields": {

663 + "tool_calls": {

664 + "open_pattern": r"<tool_call>\s*<function=(?P<name>\w+)>",

665 + "close": "</tool_call>",

666 + "repeats": true,

667 + "content": "xml-inline",

668 + "content_args": {

669 + "tag_pattern": r"<parameter=(?P<key>\w+)>\s*(?P<value>.*?)\s*</parameter>",

670 + "merge_duplicates": true,

671 + },

672 + "transform": {"type": "function", "function": {"name": "{name}", "arguments": "{content}"}},

673 + },

674 + },

675 + }));

676 + let tools = [Tool {

677 + name: "set_alarm".to_string(),

678 + description: None,

679 + parameters: json!({

680 + "type": "object",

681 + "properties": {

682 + "hour": {"type": "integer"},

683 + "enabled": {"type": "boolean"},

684 + "label": {"type": "string"},

685 + "weekday": {"type": "integer"},

686 + },

687 + }),

688 + strict: None,

689 + }];

690 + let text = "<tool_call>\n<function=set_alarm>\n<parameter=hour>\n7\n</parameter>\n<parameter=enabled>\ntrue\n</parameter>\n<parameter=label>\nwake up\n</parameter>\n<parameter=weekday>1</parameter><parameter=weekday>2</parameter>\n</function>\n</tool_call>";

691 + assert_eq!(

692 + message(&parse_events(&template, &tools, "", &[text]).unwrap()),

693 + json!({"tool_calls": [{"name": "set_alarm", "arguments": {"hour": 7, "enabled": true, "label": "wake up", "weekday": [1, 2]}}]})

694 + );

695 + // Without tools, values stay strings.

696 + assert_eq!(

697 + message(&parse_events(&template, &[], "", &[text]).unwrap()),

698 + json!({"tool_calls": [{"name": "set_alarm", "arguments": {"hour": "7", "enabled": "true", "label": "wake up", "weekday": ["1", "2"]}}]})

699 + );

700 +}

701 +

702 +#[test]

703 +fn stripped_reasoning_whitespace_keeps_its_tokens() {

704 + let template = compile(gemma4_template());

705 + let mut parser = HfUnifiedParser::new(template, &[], tokenizer());

706 + parser.initialize(&[]).unwrap();

707 + let text = "<|channel>thought\n \n plan \n<channel|>";

708 + let mut output = UnifiedParserOutput::default();

709 + for (index, c) in text.chars().enumerate() {

710 + let piece = DecodedText {

711 + text: c.to_string(),

712 + attributions: [TokenAttribution {

713 + token_id: u32::try_from(index).unwrap(),

714 + anchor: TokenAnchor::Visible { byte_offset: 0 },

715 + }]

716 + .into_iter()

717 + .collect(),

718 + };

719 + parser.parse_into(piece, &mut output).unwrap();

720 + }

721 + output.append(parser.finish().unwrap());

722 + let (reasoning, tokens) =

723 + output

724 + .events

725 + .iter()

726 + .fold((String::new(), 0), |(text, tokens), event| match event {

727 + UnifiedParserEvent::Reasoning(piece) => {

728 + (text + &piece.text, tokens + piece.attributions.len())

729 + }

730 + _ => (text, tokens),

731 + });

732 + assert_eq!(reasoning, "plan");

733 + // Every character between the markers is a reasoning token, stripped or not.

734 + assert_eq!(tokens, " \n plan \n".chars().count());

735 +}

736 +

737 +#[test]

738 +fn reset_returns_uncommitted_text() {

739 + let template = compile(gemma4_template());

740 + let mut parser = HfUnifiedParser::new(template, &[], tokenizer());

741 + parser.initialize(&[]).unwrap();

742 + let mut output = UnifiedParserOutput::default();

743 + parser

744 + .parse_into(

745 + DecodedText::unattributed("Hi <|tool_call>call:f{a:<|\"|>x"),

746 + &mut output,

747 + )

748 + .unwrap();

749 + assert_eq!(parser.reset(), "<|tool_call>call:f{a:<|\"|>x");

750 +}

751 +

752 +#[test]

753 +fn malformed_tool_arguments_fail_the_parser() {

754 + let template = compile(gemma4_template());

755 + let error =

756 + parse_events(&template, &[], "", &["<|tool_call>call:f{a:}<tool_call|>"]).unwrap_err();

757 + assert!(

758 + matches!(error, UnifiedParserError::ParsingFailed { .. }),

759 + "{error:?}"

760 + );

761 +}

762 +

763 +#[test]

764 +fn template_validation_errors() {

765 + let error = |template: Value| ResponseTemplate::from_json(&template).unwrap_err();

766 + expect_test::expect![[r#"

767 + [

768 + Invalid {

769 + message: "unsupported response_template version: 2",

770 + },

771 + Invalid {

772 + message: "At most one field may omit 'open'/'open_pattern' (that field becomes the implicit-open / leftover sink). Found: content, thinking",

773 + },

774 + Invalid {

775 + message: "response_template must define 'start_anchor' or 'start_anchor_pattern'.",

776 + },

777 + Invalid {

778 + message: "Field 'content': 'join' requires 'repeats': true",

779 + },

780 + Invalid {

781 + message: "Field 'tool_calls': open_pattern/close_pattern declares named group(s) [\"name\"], but the field has no 'transform'. Named captures are only surfaced through a 'transform' template (where they appear alongside 'content'). Either add a 'transform' that uses the captures, or remove the named groups from the pattern.",

782 + },

783 + Invalid {

784 + message: "Field 'tool_calls': transform string \"name: {content}\" mixes a {placeholder} with literal text. Use either a whole-string placeholder (e.g. \"{content}\") or a plain literal; string interpolation is not supported.",

785 + },

786 + Invalid {

787 + message: "Field 'content': 'open' literals cannot be empty strings",

788 + },

789 + Unsupported {

790 + message: "Field 'count': only 'content', 'reasoning_content'/'thinking', and 'tool_calls' fields can be reported",

791 + },

792 + Unsupported {

793 + message: "Field 'thinking': text and reasoning fields must use the 'text' content parser without a transform",

794 + },

795 + Invalid {

796 + message: "response_template.fields.content.content: unknown variant `yaml`, expected one of `text`, `int`, `float`, `bool`, `json`, `xml-inline`, `kv-lines`",

797 + },

798 + Invalid {

799 + message: "response_template.fields.content.repeats: invalid type: string \"yes\", expected a boolean",

800 + },

801 + Invalid {

802 + message: "Field 'content': cannot specify both 'open' and 'open_pattern'",

803 + },

804 + Invalid {

805 + message: "Field 'tool_calls': transform_each is set but no transform was provided",

806 + },

807 + Unsupported {

808 + message: "Field 'tool_calls': 'join' requires each match to parse to a string",

809 + },

810 + Invalid {

811 + message: "Field 'tool_calls': transform placeholder '{id}' is neither 'content' nor a named group of open_pattern",

812 + },

813 + ]

814 + "#]]

815 + .assert_debug_eq(&[

816 + error(json!({"version": 2, "start_anchor": "a", "fields": {"content": {}}})),

817 + error(json!({"start_anchor": "a", "fields": {"content": {}, "thinking": {}}})),

818 + error(json!({"fields": {"content": {}}})),

819 + error(json!({"start_anchor": "a", "fields": {"content": {"open": "<x>", "join": ""}}})),

820 + error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open_pattern": r"<t (?P<name>\w+)>", "close": "</t>"}}})),

821 + error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open": "<t>", "transform": {"label": "name: {content}"}}}})),

822 + error(json!({"start_anchor": "a", "fields": {"content": {"open": [""]}}})),

823 + error(json!({"start_anchor": "a", "fields": {"count": {"open": "<n>", "content": "int"}}})),

824 + error(json!({"start_anchor": "a", "fields": {"thinking": {"open": "<n>", "content": "json"}}})),

825 + error(json!({"start_anchor": "a", "fields": {"content": {"content": "yaml"}}})),

826 + error(json!({"start_anchor": "a", "fields": {"content": {"repeats": "yes"}}})),

827 + error(json!({"start_anchor": "a", "fields": {"content": {"open": "<x>", "open_pattern": "<y>"}}})),

828 + error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open": "<t>", "transform_each": true}}})),

829 + error(json!({"start_anchor": "a", "fields": {"tool_calls": {"open": "<t>", "repeats": true, "join": ""}}})),

830 + error(json!({

831 + "start_anchor": "a",

832 + "fields": {"tool_calls": {

833 + "open_pattern": r"<t (?P<name>\w+)>",

834 + "close_pattern": r"</t (?P<id>\d+)>",

835 + "content": "json",

836 + "transform": {"function": {"name": "{name}", "arguments": "{content}", "id": "{id}"}},

837 + }},

838 + })),

839 + ]);

840 +}

841 +

842 +/// Cases where the Rust parser intentionally differs from Transformers, keyed by

843 +/// generated text.

844 +const DIVERGENCES: &[(&str, &str)] = &[

845 + (

846 + // A field without `repeats` keeps only its last occurrence in Transformers;

847 + // streamed text cannot be retracted, so every occurrence is reported.

848 + r#"Before <|tool_call>call:f{x:<|"|>a, b<|"|>,y:[1,2.5,-3],z:{w:true}}<tool_call|> after"#,

849 + r#"{"content": "Before after", "tool_calls": [{"name": "f", "arguments": {"x": "a, b", "y": [1, 2.5, -3], "z": {"w": true}}}]}"#,

850 + ),

851 + (

852 + // Arguments use the tool parsers' shared conversion, which keeps a JSON

853 + // number's spelling; Transformers turns `1e3` into the integer 1000.

854 + "<tool_call>\n<function=set_alarm>\nhour: seven\nratio: 1e3\nenabled: nope\n</tool_call>",

855 + r#"{"tool_calls": [{"name": "set_alarm", "arguments": {"hour": "seven", "ratio": 1000.0, "enabled": "nope"}}]}"#,

856 + ),

857 +];

858 +

859 +/// Compare against `fixtures/differential.json`, generated from Transformers

860 +/// `parse_response` by `fixtures/generate_differential.py`.

861 +#[test]

862 +fn differential_against_transformers() {

863 + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))

864 + .join("src/unified/hf/fixtures/differential.json");

865 + let corpus: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap();

866 + let templates: BTreeMap<&str, Arc<ResponseTemplate>> = corpus["templates"]

867 + .as_object()

868 + .unwrap()

869 + .iter()

870 + .map(|(name, template)| {

871 + (

872 + name.as_str(),

873 + Arc::new(ResponseTemplate::from_json(template).unwrap()),

874 + )

875 + })

876 + .collect();

877 +

878 + for case in corpus["cases"].as_array().unwrap() {

879 + let template = &templates[case["template"].as_str().unwrap()];

880 + let prefix = case["prefix"].as_str().unwrap();

881 + let text = case["text"].as_str().unwrap();

882 + let tools: Vec<Tool> = case["tools"]

883 + .as_array()

884 + .into_iter()

885 + .flatten()

886 + .map(|tool| Tool {

887 + name: tool["function"]["name"].as_str().unwrap().to_string(),

888 + description: None,

889 + parameters: tool["function"]["parameters"].clone(),

890 + strict: None,

891 + })

892 + .collect();

893 + let expected = match DIVERGENCES.iter().find(|(divergent, _)| *divergent == text) {

894 + Some((_, expected)) => serde_json::from_str(expected).unwrap(),

895 + None => case["expected"].clone(),

896 + };

897 +

898 + for chunks in [vec![text], split_by_chars(text, 1)] {

899 + let actual = parse_events(template, &tools, prefix, &chunks);

900 + match (&actual, expected.get("error")) {

901 + (Err(_), Some(_)) => {}

902 + (Ok(events), None) => assert_eq!(message(events), expected, "{text:?}"),

903 + _ => panic!("{text:?}: expected {expected}, got {actual:?}"),

904 + }

905 + }

906 + }

907 +}

908 +

909 +#[test]

910 +fn tool_call_value_shapes() {

911 + // No transform: the parsed JSON is the tool call itself.

912 + let template = compile(json!({

913 + "start_anchor": "<|assistant|>",

914 + "fields": {"tool_calls": {"open": "<tool_call>", "close": "</tool_call>", "content": "json"}},

915 + }));

916 + let parse =

917 + |text: &str| parse_events(&template, &[], "", &[text]).map(|events| message(&events));

918 +

919 + assert_eq!(

920 + parse(r#"<tool_call>{"type": "function", "function": {"name": "f", "arguments": {"a": 1}}}</tool_call>"#).unwrap(),

921 + json!({"tool_calls": [{"name": "f", "arguments": {"a": 1}}]})

922 + );

923 + assert_eq!(

924 + parse(r#"<tool_call>[{"name": "f", "arguments": null}, {"name": "g", "arguments": "raw"}]</tool_call>"#).unwrap(),

925 + json!({"tool_calls": [{"name": "f", "arguments": {}}, {"name": "g", "arguments": "raw"}]})

926 + );

927 + expect_test::expect![[r#"

928 + ParsingFailed {

929 + message: "tool call must be {\"function\": {\"name\": ..., \"arguments\": ...}} or {\"name\": ..., \"arguments\": ...} with a string name, got {\"name\":1}",

930 + }

931 + "#]]

932 + .assert_debug_eq(&parse(r#"<tool_call>{"name": 1}</tool_call>"#).unwrap_err());

933 +}

rust/src/parser/src/unified/hf/transform.rs new

+263

@@ -0,0 +1,263 @@

1 +// SPDX-License-Identifier: Apache-2.0

2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +

4 +//! Transform templates: restructure a parsed region value into the output shape.

5 +//!

6 +//! Port of `_apply_transform`, `validate_transform_strings`, and `process_field`

7 +//! from `content_parsers.py`.

8 +

9 +use std::sync::LazyLock;

10 +

11 +use regex_automata::meta::Regex;

12 +use serde_json::{Map, Value};

13 +

14 +use super::pattern::Captures;

15 +use super::{Result, invalid};

16 +use crate::unified::{self, parsing_failed};

17 +

18 +/// `\{(\w+(?:\.\w+)*)\}`: a whole-string placeholder such as `{content}` or

19 +/// `{content.args}`.

20 +static PLACEHOLDER: LazyLock<Regex> =

21 + LazyLock::new(|| Regex::new(r"\{\w+(?:\.\w+)*\}").expect("valid regex"));

22 +

23 +/// A compiled transform template.

24 +#[derive(Debug, Clone, PartialEq)]

25 +pub(super) enum Transform {

26 + Object(Vec<(String, Transform)>),

27 + Array(Vec<Transform>),

28 + /// A dotted placeholder path, e.g. `["content", "args"]`.

29 + Placeholder(Vec<String>),

30 + /// Any other JSON value, copied verbatim.

31 + Literal(Value),

32 +}

33 +

34 +impl Transform {

35 + /// Compile a transform template, rejecting any string that mixes a

36 + /// `{name}` placeholder with literal text.

37 + pub fn compile(scope: &str, template: &Value) -> Result<Self> {

38 + Ok(match template {

39 + Value::Object(object) => Self::Object(

40 + object

41 + .iter()

42 + .map(|(key, value)| Ok((key.clone(), Self::compile(scope, value)?)))

43 + .collect::<Result<_>>()?,

44 + ),

45 + Value::Array(values) => Self::Array(

46 + values.iter().map(|value| Self::compile(scope, value)).collect::<Result<_>>()?,

47 + ),

48 + Value::String(string) => match PLACEHOLDER.find(string) {

49 + Some(found) if found.range() == (0..string.len()) => Self::Placeholder(

50 + string[1..string.len() - 1].split('.').map(str::to_string).collect(),

51 + ),

52 + Some(_) => {

53 + return Err(invalid!(

54 + "{scope}: transform string {string:?} mixes a {{placeholder}} with literal text. \

55 + Use either a whole-string placeholder (e.g. \"{{content}}\") or a plain literal; \

56 + string interpolation is not supported."

57 + ));

58 + }

59 + None => Self::Literal(template.clone()),

60 + },

61 + other => Self::Literal(other.clone()),

62 + })

63 + }

64 +

65 + /// Return the placeholder path found at `key_path` inside this template, if any.

66 + pub fn placeholder_at(&self, key_path: &[&str]) -> Option<&[String]> {

67 + match (self, key_path) {

68 + (Self::Placeholder(path), []) => Some(path),

69 + (Self::Object(entries), [key, rest @ ..]) => entries

70 + .iter()

71 + .find(|(entry, _)| entry == key)

72 + .and_then(|(_, transform)| transform.placeholder_at(rest)),

73 + _ => None,

74 + }

75 + }

76 +

77 + /// Root names of every placeholder in this template.

78 + pub fn placeholder_roots(&self) -> Box<dyn Iterator<Item = &str> + '_> {

79 + match self {

80 + Self::Object(entries) => {

81 + Box::new(entries.iter().flat_map(|(_, transform)| transform.placeholder_roots()))

82 + }

83 + Self::Array(transforms) => {

84 + Box::new(transforms.iter().flat_map(Self::placeholder_roots))

85 + }

86 + Self::Placeholder(path) => Box::new(path.first().map(String::as_str).into_iter()),

87 + Self::Literal(_) => Box::new(std::iter::empty()),

88 + }

89 + }

90 +

91 + /// Recursively instantiate this template against `scope`.

92 + pub fn apply(&self, scope: &Map<String, Value>) -> unified::Result<Value> {

93 + Ok(match self {

94 + Self::Object(entries) => Value::Object(

95 + entries

96 + .iter()

97 + .map(|(key, transform)| Ok((key.clone(), transform.apply(scope)?)))

98 + .collect::<unified::Result<_>>()?,

99 + ),

100 + Self::Array(transforms) => Value::Array(

101 + transforms

102 + .iter()

103 + .map(|transform| transform.apply(scope))

104 + .collect::<unified::Result<_>>()?,

105 + ),

106 + Self::Literal(value) => value.clone(),

107 + Self::Placeholder(path) => {

108 + let dotted = path.join(".");

109 + let (root, keys) = path.split_first().expect("placeholder path is non-empty");

110 + let Some(mut value) = scope.get(root) else {

111 + let available: Vec<_> = scope.keys().collect();

112 + return Err(parsing_failed!(

113 + "transform placeholder '{{{dotted}}}' is not defined. Available: {available:?}"

114 + ));

115 + };

116 + for key in keys {

117 + let Some(object) = value.as_object() else {

118 + return Err(parsing_failed!(

119 + "transform placeholder '{{{dotted}}}' cannot index into {} at '{key}'",

120 + json_type_name(value)

121 + ));

122 + };

123 + let Some(next) = object.get(key) else {

124 + let available: Vec<_> = object.keys().collect();

125 + return Err(parsing_failed!(

126 + "transform placeholder '{{{dotted}}}' is missing key '{key}'. Available: {available:?}"

127 + ));

128 + };

129 + value = next;

130 + }

131 + value.clone()

132 + }

133 + })

134 + }

135 +}

136 +

137 +/// A field's transform: applied to the parsed value, or to each of its elements.

138 +#[derive(Debug, Clone, PartialEq)]

139 +pub(super) struct FieldTransform {

140 + pub template: Transform,

141 + /// `transform_each`: the parsed content must be a list of dicts, and the

142 + /// template is applied to each element (with the element's keys unpacked into

143 + /// the template scope, alongside any regex captures).

144 + pub each: bool,

145 +}

146 +

147 +impl FieldTransform {

148 + /// Apply the transform to a parsed region value.

149 + pub fn apply(

150 + &self,

151 + field_name: &str,

152 + parsed: Value,

153 + captures: &Captures,

154 + ) -> unified::Result<Value> {

155 + if !self.each {

156 + let mut scope = captures.clone();

157 + scope.insert("content".to_string(), parsed);

158 + return self.template.apply(&scope);

159 + }

160 +

161 + let Value::Array(items) = parsed else {

162 + return Err(parsing_failed!(

163 + "Field '{field_name}': transform_each requires the parsed content to be an array, got {}.",

164 + json_type_name(&parsed)

165 + ));

166 + };

167 + items

168 + .into_iter()

169 + .map(|item| {

170 + let Value::Object(item) = item else {

171 + return Err(parsing_failed!(

172 + "Field '{field_name}': transform_each requires each array element to be an object, got {}.",

173 + json_type_name(&item)

174 + ));

175 + };

176 + let mut scope = captures.clone();

177 + scope.extend(item);

178 + self.template.apply(&scope)

179 + })

180 + .collect::<unified::Result<_>>()

181 + .map(Value::Array)

182 + }

183 +}

184 +

185 +/// JSON type name of a value, for error messages.

186 +fn json_type_name(value: &Value) -> &'static str {

187 + match value {

188 + Value::Null => "null",

189 + Value::Bool(_) => "boolean",

190 + Value::Number(_) => "number",

191 + Value::String(_) => "string",

192 + Value::Array(_) => "array",

193 + Value::Object(_) => "object",

194 + }

195 +}

196 +

197 +#[cfg(test)]

198 +mod tests {

199 + use serde_json::json;

200 +

201 + use super::*;

202 +

203 + fn scope(value: Value) -> Map<String, Value> {

204 + value.as_object().unwrap().clone()

205 + }

206 +

207 + #[test]

208 + fn dotted_placeholders_descend_into_objects() {

209 + let transform = Transform::compile(

210 + "test",

211 + &json!({"type": "function", "function": {"name": "{content.name}", "arguments": "{content.args}"}}),

212 + )

213 + .unwrap();

214 + let value = transform

215 + .apply(&scope(json!({"content": {"name": "f", "args": {"a": 1}}})))

216 + .unwrap();

217 + assert_eq!(

218 + value,

219 + json!({"type": "function", "function": {"name": "f", "arguments": {"a": 1}}})

220 + );

221 + assert_eq!(

222 + transform.placeholder_at(&["function", "name"]),

223 + Some(["content".to_string(), "name".to_string()].as_slice())

224 + );

225 + }

226 +

227 + #[test]

228 + fn placeholder_errors_match_transformers() {

229 + let apply = |template: Value, scope_value: Value| {

230 + Transform::compile("test", &template)

231 + .unwrap()

232 + .apply(&scope(scope_value))

233 + .unwrap_err()

234 + };

235 + expect_test::expect![[r#"

236 + ParsingFailed {

237 + message: "transform placeholder '{name}' is not defined. Available: [\"content\"]",

238 + }

239 + "#]]

240 + .assert_debug_eq(&apply(json!("{name}"), json!({"content": 1})));

241 + expect_test::expect![[r#"

242 + ParsingFailed {

243 + message: "transform placeholder '{content.a}' cannot index into number at 'a'",

244 + }

245 + "#]]

246 + .assert_debug_eq(&apply(json!("{content.a}"), json!({"content": 1})));

247 + expect_test::expect![[r#"

248 + ParsingFailed {

249 + message: "transform placeholder '{content.b}' is missing key 'b'. Available: [\"a\"]",

250 + }

251 + "#]]

252 + .assert_debug_eq(&apply(json!("{content.b}"), json!({"content": {"a": 1}})));

253 + }

254 +

255 + #[test]

256 + fn interpolation_is_rejected_at_load() {

257 + assert!(Transform::compile("test", &json!({"name": "fn_{name}"})).is_err());

258 + assert_eq!(

259 + Transform::compile("test", &json!("no placeholder")).unwrap(),

260 + Transform::Literal(json!("no placeholder"))

261 + );

262 + }

263 +}

rust/src/parser/src/unified/mod.rs

+9 -2

@@ -5,12 +5,14 @@

6 mod combined;

7 mod gemma4;

8 +mod hf;

9 mod hy;

10 mod inkling;

11 mod kimi_k3;

13 pub use combined::CombinedParser;

14 pub use gemma4::Gemma4UnifiedParser;

15 +pub use hf::{HfTemplateError, HfUnifiedParser, ResponseTemplate};

16 pub use hy::{HyV3UnifiedParser, HyV4UnifiedParser};

17 pub use inkling::InklingUnifiedParser;

18 pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser};

@@ -271,8 +273,13 @@ pub trait UnifiedParser: Send {

273 #[derive(Debug, Error, Macro)]

274 #[thiserror_ext(macro(path = "crate::unified", mangle))]

275 pub enum UnifiedParserError {

274 - #[error("combined parser is constructed from split parser instances")]

275 - CombinedParserConstructor,

276 + /// The parser is built from inputs other than tools and a tokenizer, so

277 + /// [`UnifiedParser::create`] cannot construct it.

278 + #[error("the `{parser}` unified parser is built from {built_from}, not by name")]

279 + NoNamedConstructor {

280 + parser: &'static str,

281 + built_from: &'static str,

282 + },

283 #[error("tokenizer is missing unified parser token `{token}`")]

284 MissingToken { token: String },

285 #[error("unified parser parsing failed: {message}")]

rust/src/text/src/backend/hf/config.rs

@@ -23,6 +23,9 @@ pub struct HfTokenizerConfig {

23 /// containing "Tiktoken" which can be used as a hint for backend

24 /// selection.

25 pub tokenizer_class: Option<String>,

26 + /// The Hugging Face `response_template`: a declarative description of the

27 + /// model's output protocol, executed by the `hf` unified parser.

28 + pub response_template: Option<serde_json::Value>,

29 }

31 /// Hugging Face named special tokens may be serialized as a string or an