@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 2899dca843

[Model] Add Kimi K3 support: Rust frontend [1/2] (#50104)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Jul 29, 2026 at 05:31 UTC 2899dca8432d40632987b0ec24253a8fe6df2710
35 files changed +3003 -39
pyproject.toml
+2 -3
@@ -129,10 +129,9 @@ extend-exclude = ["tests/models/fixtures/*", "tests/prompts/*", "tests/tokenizer
129 "tests/entrypoints/speech_to_text/transcription/test_transcription_validation.py",
130 "docs/governance/process.md", "docs/assets/contributing/vllm_bench_serve_timeline.html",
131 "tests/v1/engine/test_fast_incdec_prefix_err.py", ".git/*", "csrc/cpu/sgl-kernels/*",
132 - "rust/src/chat/src/renderer/deepseek_v32/fixtures/*",
133 - "rust/src/parser/src/tool/gemma4.rs", "rust/src/parser/src/unified/gemma4.rs",
132 + "rust/src/chat/src/renderer/deepseek_v32/fixtures/*", "rust/src/parser/**",
133 "rust/src/text/src/output/decoded.rs",
135 - "rust/src/tokenizer/src/incremental.rs", "rust/src/parser/src/reasoning/tests.rs"]
134 + "rust/src/tokenizer/src/incremental.rs"]
135 ignore-hidden = false
136
137 [tool.typos.default]
rust/Cargo.lock
+1 -1
@@ -2220,7 +2220,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
2220 [[package]]
2221 name = "llm-multimodal"
2222 version = "1.7.1"
2223 -source = "git+https://github.com/smg-project/llm-multimodal?rev=5390032d6dc8a3e6fdc83acd320260367eb4b9b5#5390032d6dc8a3e6fdc83acd320260367eb4b9b5"
2223 +source = "git+https://github.com/smg-project/llm-multimodal?rev=15adba5e025d8636ba4a334fb379b1371f6196a1#15adba5e025d8636ba4a334fb379b1371f6196a1"
2224 dependencies = [
2225 "anyhow",
2226 "base64 0.22.1",
rust/Cargo.toml
+1 -1
@@ -59,7 +59,7 @@ indexmap = "2.13.0"
59 indicatif = "0.18.4"
60 itertools = "0.14.0"
61 libc = "0.2.177"
62 -llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "5390032d6dc8a3e6fdc83acd320260367eb4b9b5", default-features = false, features = ["native-tls"] }
62 +llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "15adba5e025d8636ba4a334fb379b1371f6196a1", default-features = false, features = ["native-tls"] }
63 mimalloc = "0.1.52"
64 minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
65 minijinja-contrib = { version = "2.0", features = ["pycompat"] }
rust/src/chat/src/backend/hf.rs
+2 -1
@@ -20,7 +20,7 @@ use crate::output::{
20 use crate::renderer::hf::{HfChatRenderer, MultimodalRenderInfo};
21 use crate::renderer::{
22 DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer, HarmonyChatRenderer,
23 - InklingChatRenderer,
23 + InklingChatRenderer, KimiK3ChatRenderer,
24 };
25 use crate::request::ChatRequest;
26 use crate::{DynChatOutputProcessor, RendererSelection};
@@ -73,6 +73,7 @@ impl HfChatBackend {
73 RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
74 RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
75 RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
76 + RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(tokenizer.clone())),
77 };
78
79 info!(
rust/src/chat/src/lib.rs
+10 -3
@@ -33,7 +33,8 @@ pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
33 pub use renderer::hf::ChatTemplateContentFormatOption;
34 pub use renderer::{
35 ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DynChatRenderer,
36 - HarmonyChatRenderer, InklingChatRenderer, RenderedPrompt, RendererSelection,
36 + HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt,
37 + RendererSelection,
38 };
39 pub use request::{
40 ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
@@ -353,6 +354,12 @@ mod tests {
354 .unwrap();
355 }
356
357 + #[test]
358 + fn validate_parser_overrides_accepts_explicit_kimi_k3() {
359 + let selection = ParserSelection::Explicit("kimi_k3".to_string());
360 + validate_parser_overrides(&selection, &selection).unwrap();
361 + }
362 +
363 #[test]
364 fn validate_parser_overrides_accepts_auto_and_none() {
365 validate_parser_overrides(&ParserSelection::Auto, &ParserSelection::None).unwrap();
@@ -366,7 +373,7 @@ mod tests {
373 )
374 .unwrap_err();
375
369 - expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
376 + expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, granite4, hermes, hy_v3, inkling, internlm, kimi_k2, kimi_k3, llama3_json, llama4_json, minimax_m2, minimax_m3, mistral, phi4_mini_json, qwen3_coder, qwen3_xml, seed_oss)"].assert_eq(&error.to_report_string());
377 }
378
379 #[test]
@@ -377,6 +384,6 @@ mod tests {
384 )
385 .unwrap_err();
386
380 - expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
387 + expect_test::expect!["reasoning parser `definitely_missing_reasoning_parser` is not registered (choose from: cohere_cmd, deepseek_r1, deepseek_v3, deepseek_v4, gemma4, glm45, inkling, kimi, kimi_k2, kimi_k3, minimax_m2, minimax_m3, nemotron_v3, qwen3, seed_oss, step3, step3p5)"].assert_eq(&error.to_report_string());
388 }
389 }
rust/src/chat/src/parser/reasoning/mod.rs
+2
@@ -27,6 +27,7 @@ pub mod names {
27 pub const GLM45: &str = "glm45";
28 pub const KIMI: &str = "kimi";
29 pub const KIMI_K2: &str = "kimi_k2";
30 + pub const KIMI_K3: &str = "kimi_k3";
31 pub const MINIMAX_M2: &str = "minimax_m2";
32 pub const MINIMAX_M3: &str = "minimax_m3";
33 pub const NEMOTRON_V3: &str = "nemotron_v3";
@@ -68,6 +69,7 @@ impl ReasoningParserFactory {
69 .register_parser::<Glm45ReasoningParser>(names::GLM45)
70 .register_parser::<KimiReasoningParser>(names::KIMI)
71 .register_parser::<KimiK2ReasoningParser>(names::KIMI_K2)
72 + .register_unified_dummy(names::KIMI_K3)
73 .register_parser::<MiniMaxM2ReasoningParser>(names::MINIMAX_M2)
74 .register_parser::<MiniMaxM3ReasoningParser>(names::MINIMAX_M3)
75 .register_parser::<NemotronV3ReasoningParser>(names::NEMOTRON_V3)
rust/src/chat/src/parser/tool/mod.rs
+2
@@ -33,6 +33,7 @@ pub mod names {
33 // also routes to `Internlm2ToolParser` despite the version-agnostic name.
34 pub const INTERNLM: &str = "internlm";
35 pub const KIMI_K2: &str = "kimi_k2";
36 + pub const KIMI_K3: &str = "kimi_k3";
37 pub const LLAMA3_JSON: &str = "llama3_json";
38 pub const LLAMA4_JSON: &str = "llama4_json";
39 pub const MINIMAX_M2: &str = "minimax_m2";
@@ -78,6 +79,7 @@ impl ToolParserFactory {
79 .register_parser::<HyV3ToolParser>(names::HY_V3)
80 .register_parser::<Internlm2ToolParser>(names::INTERNLM)
81 .register_parser::<KimiK2ToolParser>(names::KIMI_K2)
82 + .register_unified_dummy(names::KIMI_K3)
83 .register_parser::<Llama3JsonToolParser>(names::LLAMA3_JSON)
84 .register_parser::<Llama3JsonToolParser>(names::LLAMA4_JSON)
85 .register_parser::<MinimaxM2ToolParser>(names::MINIMAX_M2)
rust/src/chat/src/parser/unified.rs
+24 -2
@@ -5,7 +5,9 @@
5
6 use std::sync::LazyLock;
7
8 -pub use vllm_parser::unified::{Gemma4UnifiedParser, InklingUnifiedParser, UnifiedParser};
8 +pub use vllm_parser::unified::{
9 + Gemma4UnifiedParser, InklingUnifiedParser, KimiK3UnifiedParser, UnifiedParser,
10 +};
11 use vllm_tokenizer::DynTokenizer;
12
13 use crate::parser::ParserFactory;
@@ -15,6 +17,7 @@ use crate::request::ChatTool;
17 pub mod names {
18 pub const GEMMA4: &str = "gemma4";
19 pub const INKLING: &str = "inkling";
20 + pub const KIMI_K3: &str = "kimi_k3";
21 }
22
23 /// Constructor signature for one registered unified parser implementation.
@@ -39,11 +42,14 @@ impl UnifiedParserFactory {
42
43 factory.register_parser::<Gemma4UnifiedParser>(names::GEMMA4);
44 factory.register_parser::<InklingUnifiedParser>(names::INKLING);
45 + factory.register_parser::<KimiK3UnifiedParser>(names::KIMI_K3);
46
47 factory
48 .register_pattern("gemma-4", names::GEMMA4)
49 .register_pattern("gemma4", names::GEMMA4)
46 - .register_pattern("inkling", names::INKLING);
50 + .register_pattern("inkling", names::INKLING)
51 + .register_pattern("kimi-k3", names::KIMI_K3)
52 + .register_pattern("kimi_k3", names::KIMI_K3);
53
54 factory
55 }
@@ -121,4 +127,20 @@ mod tests {
127 );
128 factory.create(names::INKLING, &[], Arc::new(inkling_tokenizer())).unwrap();
129 }
130 +
131 + #[test]
132 + fn factory_registers_kimi_k3() {
133 + let factory = UnifiedParserFactory::new();
134 + let tokenizer = TestTokenizer::new()
135 + .with_regular_token("<|open|>", 1001)
136 + .with_regular_token("<|close|>", 1002)
137 + .with_regular_token("<|sep|>", 1003);
138 +
139 + assert!(factory.contains(names::KIMI_K3));
140 + assert_eq!(
141 + factory.resolve_name_for_model("moonshotai/Kimi-K3"),
142 + Some(names::KIMI_K3)
143 + );
144 + factory.create(names::KIMI_K3, &[], Arc::new(tokenizer)).unwrap();
145 + }
146 }
rust/src/chat/src/renderer/deepseek_v32/tests.rs
+1 -1
@@ -59,7 +59,7 @@ fn fixture_request(input_name: &str) -> ChatRequest {
59
60 fn deepseek_fixture_options() -> FixtureRequestOptions {
61 FixtureRequestOptions {
62 - enable_thinking: true,
62 + enable_thinking: Some(true),
63 no_generation_prompt_when_last_assistant: true,
64 }
65 }
rust/src/chat/src/renderer/deepseek_v4/tests.rs
+1 -1
@@ -27,7 +27,7 @@ fn fixture_request(input_name: &str) -> ChatRequest {
27
28 fn deepseek_fixture_options() -> FixtureRequestOptions {
29 FixtureRequestOptions {
30 - enable_thinking: true,
30 + enable_thinking: Some(true),
31 no_generation_prompt_when_last_assistant: true,
32 }
33 }
rust/src/chat/src/renderer/harmony/tests.rs
+1 -1
@@ -22,7 +22,7 @@ fn fixture_request(input_name: &str) -> ChatRequest {
22 fixture_chat_request(
23 &fixture_path(input_name),
24 FixtureRequestOptions {
25 - enable_thinking: false,
25 + enable_thinking: Some(false),
26 no_generation_prompt_when_last_assistant: false,
27 },
28 )
rust/src/chat/src/renderer/inkling/tests.rs
+1 -1
@@ -105,7 +105,7 @@ fn fixture_request(name: &str) -> ChatRequest {
105
106 fn inkling_fixture_options() -> FixtureRequestOptions {
107 FixtureRequestOptions {
108 - enable_thinking: false,
108 + enable_thinking: Some(false),
109 no_generation_prompt_when_last_assistant: false,
110 }
111 }
rust/src/chat/src/renderer/kimi_k3/encoding.rs new
+608
@@ -0,0 +1,608 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +//! Kimi K3 XTML prompt renderer.
5 +//!
6 +//! Port of Moonshot remote-code `encoding_k3.py::build_chat_segments()`.
7 +
8 +use std::collections::HashMap;
9 +
10 +use serde_json::{Map, Value, json};
11 +use vllm_tokenizer::Tokenizer;
12 +
13 +use crate::error::{Error, Result};
14 +use crate::request::{
15 + ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice,
16 +};
17 +use crate::{AssistantContentBlock, AssistantToolCall};
18 +
19 +pub(super) const OPEN: &str = "<|open|>";
20 +pub(super) const CLOSE: &str = "<|close|>";
21 +pub(super) const SEP: &str = "<|sep|>";
22 +pub(super) const END_OF_MSG: &str = "<|end_of_msg|>";
23 +pub(super) const IMAGE_PLACEHOLDER: &str = "<|media_pad|>";
24 +
25 +const DEFAULT_THINKING_EFFORT: &str = "max";
26 +const VALID_THINKING_EFFORTS: &[&str] = &["low", "high", "max"];
27 +
28 +/// K3 prompt encoder preserving Python's per-segment tokenization boundaries.
29 +pub(super) struct K3TokenWriter<'a> {
30 + tokenizer: &'a dyn Tokenizer,
31 + token_ids: Vec<u32>,
32 +}
33 +
34 +impl<'a> K3TokenWriter<'a> {
35 + pub(super) fn new(tokenizer: &'a dyn Tokenizer) -> Self {
36 + Self {
37 + tokenizer,
38 + token_ids: Vec::new(),
39 + }
40 + }
41 +
42 + /// Encode one trusted segment with normal added-token recognition.
43 + pub(super) fn control(&mut self, text: &str) -> Result<()> {
44 + if !text.is_empty() {
45 + self.token_ids.extend(self.tokenizer.encode(text, false)?);
46 + }
47 + Ok(())
48 + }
49 +
50 + /// Encode one literal segment while bypassing every added-token matcher.
51 + pub(super) fn ordinary(&mut self, text: &str) -> Result<()> {
52 + if !text.is_empty() {
53 + self.token_ids.extend(self.tokenizer.encode_ordinary(text)?);
54 + }
55 + Ok(())
56 + }
57 +
58 + pub(super) fn finish(self) -> Vec<u32> {
59 + self.token_ids
60 + }
61 +}
62 +
63 +/// Render and tokenize one chat request using K3's segment-aware contract.
64 +pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -> Result<Vec<u32>> {
65 + let thinking = thinking_enabled(request)?;
66 + let thinking_effort = thinking.then(|| thinking_effort(request)).transpose()?;
67 + let tools = request_tools(request);
68 + let mut out = K3TokenWriter::new(tokenizer);
69 +
70 + if !tools.is_empty() {
71 + write_tool_declare(&mut out, tools, false)?;
72 + }
73 +
74 + if let Some(effort) = thinking_effort {
75 + // Preserve the checkpoint's literal guidance text: it still names
76 + // `medium`, although the validator above no longer accepts it.
77 + write_internal_system(
78 + &mut out,
79 + "thinking-effort",
80 + &format!(
81 + "`thinking_effort` guides on how much to think in your \
82 + thinking channel (not including the response channel), \
83 + supported values include `low`, `medium`, `high`, and `max`.\n\
84 + Now the system is invoked with `thinking_effort={effort}`."
85 + ),
86 + )?;
87 + }
88 +
89 + // Track prior assistant tool-call ids for tool-result reordering / naming.
90 + let mut tool_call_id_index: HashMap<String, (usize, String)> = HashMap::new();
91 + let mut pending_tool_run: Vec<(usize, ChatMessage)> = Vec::new();
92 +
93 + let flush_tool_run = |out: &mut K3TokenWriter<'_>,
94 + run: &mut Vec<(usize, ChatMessage)>,
95 + id_index: &HashMap<String, (usize, String)>|
96 + -> Result<()> {
97 + if run.is_empty() {
98 + return Ok(());
99 + }
100 +
101 + let mut resolved = Vec::with_capacity(run.len());
102 + let mut unresolved = false;
103 + for (offset, message) in run.drain(..) {
104 + let ChatMessage::ToolResponse {
105 + content,
106 + tool_call_id,
107 + } = message
108 + else {
109 + unreachable!("pending tool run only holds tool responses");
110 + };
111 + match id_index.get(&tool_call_id) {
112 + Some(&(position, ref name)) => {
113 + resolved.push((position, offset, content, Some(name.clone())));
114 + }
115 + None => {
116 + unresolved = true;
117 + resolved.push((usize::MAX, offset, content, None));
118 + }
119 + }
120 + }
121 +
122 + if unresolved {
123 + // Preserve original order when the run cannot be fully matched.
124 + resolved.sort_by_key(|item| item.1);
125 + } else {
126 + resolved.sort_by_key(|item| (item.0, item.1));
127 + }
128 +
129 + for (xtml_index, (_, _, content, name)) in resolved.into_iter().enumerate() {
130 + let tool_name = name.as_deref().ok_or_else(|| {
131 + Error::ChatTemplate(
132 + "Kimi K3 tool messages need a resolvable tool name: \
133 + carry a matching tool_call_id against a preceding \
134 + assistant tool_call"
135 + .to_string(),
136 + )
137 + })?;
138 + write_tool_message(out, tool_name, xtml_index + 1, &content)?;
139 + }
140 + Ok(())
141 + };
142 +
143 + for (message_index, message) in request.messages.iter().enumerate() {
144 + match message {
145 + ChatMessage::ToolResponse { .. } => {
146 + pending_tool_run.push((message_index, message.clone()));
147 + continue;
148 + }
149 + _ => {
150 + flush_tool_run(&mut out, &mut pending_tool_run, &tool_call_id_index)?;
151 + }
152 + }
153 +
154 + match message {
155 + // Python: role=system with a `tools` field renders a dynamic
156 + // tool-declare (`## New Tools Available`). Map that to Developer
157 + // messages that carry tools (OpenAI "developer" / system-tools).
158 + ChatMessage::Developer {
159 + content,
160 + tools: Some(local_tools),
161 + } if !local_tools.is_empty() => {
162 + write_tool_declare(&mut out, local_tools, true)?;
163 + if !content_is_empty(content) {
164 + write_role_message(&mut out, "system", None, content)?;
165 + }
166 + }
167 + ChatMessage::System { content } | ChatMessage::Developer { content, .. } => {
168 + write_role_message(&mut out, "system", None, content)?;
169 + }
170 + ChatMessage::User { content } => {
171 + write_role_message(&mut out, "user", None, content)?;
172 + }
173 + ChatMessage::Assistant { content } => {
174 + tool_call_id_index.clear();
175 + let mut call_position = 0usize;
176 + for block in content {
177 + if let AssistantContentBlock::ToolCall(call) = block {
178 + call_position += 1;
179 + if !call.id.is_empty() {
180 + tool_call_id_index
181 + .entry(call.id.clone())
182 + .or_insert((call_position, call.name.clone()));
183 + }
184 + }
185 + }
186 +
187 + write_assistant_message(&mut out, content, thinking)?;
188 + }
189 + ChatMessage::ToolResponse { .. } => unreachable!("handled above"),
190 + }
191 + }
192 + flush_tool_run(&mut out, &mut pending_tool_run, &tool_call_id_index)?;
193 +
194 + match &request.tool_choice {
195 + ChatToolChoice::Required => {
196 + write_internal_system(
197 + &mut out,
198 + "tool-choice",
199 + "The system is invoked with `tool_choice=required`.\n\
200 + You MUST call tools in the next message.",
201 + )?;
202 + }
203 + // Emit only when tools are present: Rust defaults tool_choice to None
204 + // for tool-free requests, which must not inject a tool-choice message.
205 + ChatToolChoice::None if !request.tools.is_empty() => {
206 + write_internal_system(
207 + &mut out,
208 + "tool-choice",
209 + "The system is invoked with `tool_choice=none`.\n\
210 + You MUST NOT call any tools in the next message.",
211 + )?;
212 + }
213 + ChatToolChoice::None | ChatToolChoice::Auto | ChatToolChoice::Function { .. } => {}
214 + }
215 +
216 + write_response_format(&mut out, request)?;
217 +
218 + if request.chat_options.add_generation_prompt() {
219 + write_open_tag(&mut out, "message", &[("role", "assistant")])?;
220 + write_open_tag(&mut out, if thinking { "think" } else { "response" }, &[])?;
221 + }
222 +
223 + Ok(out.finish())
224 +}
225 +
226 +fn request_tools(request: &ChatRequest) -> &[ChatTool] {
227 + // Declare tools whenever the request carries them. K3 tool-declare is
228 + // independent of tool_choice; tool_choice only injects control messages.
229 + request.tools.as_slice()
230 +}
231 +
232 +fn thinking_enabled(request: &ChatRequest) -> Result<bool> {
233 + if let Some(thinking) = request.parse_template_bool("thinking")? {
234 + return Ok(thinking);
235 + }
236 + if let Some(enable_thinking) = request.parse_template_bool("enable_thinking")? {
237 + return Ok(enable_thinking);
238 + }
239 + Ok(request
240 + .chat_options
241 + .reasoning_effort
242 + .map(|effort| effort != crate::request::ReasoningEffort::None)
243 + .unwrap_or(true))
244 +}
245 +
246 +fn thinking_effort(request: &ChatRequest) -> Result<String> {
247 + let effort = if let Some(value) = request.chat_options.template_kwargs.get("thinking_effort") {
248 + value.as_str().ok_or_else(|| {
249 + Error::ChatTemplate(format!(
250 + "template kwarg `thinking_effort` must be a string, got {value}"
251 + ))
252 + })?
253 + } else if let Some(effort) = request.chat_options.reasoning_effort {
254 + effort.as_str()
255 + } else if let Some(value) = request.chat_options.template_kwargs.get("reasoning_effort") {
256 + value.as_str().ok_or_else(|| {
257 + Error::ChatTemplate(format!(
258 + "template kwarg `reasoning_effort` must be a string, got {value}"
259 + ))
260 + })?
261 + } else {
262 + DEFAULT_THINKING_EFFORT
263 + };
264 +
265 + if !VALID_THINKING_EFFORTS.contains(&effort) {
266 + return Err(Error::ChatTemplate(format!(
267 + "unsupported thinking_effort={effort:?}; supported values are `low`, `high`, and `max`"
268 + )));
269 + }
270 + Ok(effort.to_string())
271 +}
272 +
273 +fn content_is_empty(content: &ChatContent) -> bool {
274 + match content {
275 + ChatContent::Text(text) => text.is_empty(),
276 + ChatContent::Parts(parts) => parts.iter().all(|part| match part {
277 + ChatContentPart::Text { text } => text.is_empty(),
278 + ChatContentPart::ImageUrl { .. }
279 + | ChatContentPart::VideoUrl { .. }
280 + | ChatContentPart::InputAudio { .. }
281 + | ChatContentPart::AudioUrl { .. } => false,
282 + }),
283 + }
284 +}
285 +
286 +fn write_tool_declare(
287 + out: &mut K3TokenWriter<'_>,
288 + tools: &[ChatTool],
289 + dynamic: bool,
290 +) -> Result<()> {
291 + let mut specs = Vec::with_capacity(tools.len());
292 + for tool in tools {
293 + let mut function = Map::new();
294 + function.insert(
295 + "description".to_string(),
296 + Value::String(tool.description.clone().unwrap_or_default()),
297 + );
298 + function.insert("name".to_string(), Value::String(tool.name.clone()));
299 + function.insert("parameters".to_string(), sort_json(&tool.parameters));
300 + specs.push(json!({
301 + "function": Value::Object(function),
302 + "type": "function",
303 + }));
304 + }
305 + let payload = compact_json(&sort_json(&Value::Array(specs)))?;
306 +
307 + let body = if dynamic {
308 + format!(
309 + "## New Tools Available\n\
310 + The system dynamically extends the toolset via lazy-loading.\n\
311 + You have access to all existing and extended tools.\n\
312 + Here are the specs for the extended tools.\n\n\
313 + ```json\n\
314 + {payload}\n\
315 + ```"
316 + )
317 + } else {
318 + format!(
319 + "# Tools\n\
320 + Here are the available tools, described in JSONSchema.\n\n\
321 + ```json\n\
322 + {payload}\n\
323 + ```"
324 + )
325 + };
326 +
327 + write_internal_system(out, "tool-declare", &body)
328 +}
329 +
330 +fn write_internal_system(
331 + out: &mut K3TokenWriter<'_>,
332 + message_type: &str,
333 + body: &str,
334 +) -> Result<()> {
335 + write_open_tag(
336 + out,
337 + "message",
338 + &[("role", "system"), ("type", message_type)],
339 + )?;
340 + out.ordinary(body.trim())?;
341 + write_close_tag(out, "message")?;
342 + out.control(END_OF_MSG)
343 +}
344 +
345 +fn write_role_message(
346 + out: &mut K3TokenWriter<'_>,
347 + role: &str,
348 + name: Option<&str>,
349 + content: &ChatContent,
350 +) -> Result<()> {
351 + let mut attrs = vec![("role", role.to_string())];
352 + if let Some(name) = name.filter(|name| !name.is_empty()) {
353 + attrs.push(("name", name.to_string()));
354 + }
355 + let attr_refs: Vec<(&str, &str)> = attrs.iter().map(|(k, v)| (*k, v.as_str())).collect();
356 + write_open_tag(out, "message", &attr_refs)?;
357 + write_content(out, content)?;
358 + write_close_tag(out, "message")?;
359 + out.control(END_OF_MSG)
360 +}
361 +
362 +fn write_tool_message(
363 + out: &mut K3TokenWriter<'_>,
364 + tool_name: &str,
365 + index: usize,
366 + content: &ChatContent,
367 +) -> Result<()> {
368 + let index_str = index.to_string();
369 + write_open_tag(
370 + out,
371 + "message",
372 + &[("role", "tool"), ("tool", tool_name), ("index", &index_str)],
373 + )?;
374 + write_content(out, content)?;
375 + write_close_tag(out, "message")?;
376 + out.control(END_OF_MSG)
377 +}
378 +
379 +fn write_assistant_message(
380 + out: &mut K3TokenWriter<'_>,
381 + content: &[AssistantContentBlock],
382 + thinking: bool,
383 +) -> Result<()> {
384 + write_open_tag(out, "message", &[("role", "assistant")])?;
385 +
386 + let mut reasoning = String::new();
387 + let mut response = String::new();
388 + let mut tool_calls = Vec::new();
389 + for block in content {
390 + match block {
391 + AssistantContentBlock::Reasoning { text } => reasoning.push_str(text),
392 + AssistantContentBlock::Text { text } => response.push_str(text),
393 + AssistantContentBlock::ToolCall(call) => tool_calls.push(call),
394 + }
395 + }
396 +
397 + // The think channel is structural: in thinking mode every assistant
398 + // message carries open/close tags even when there is no reasoning content.
399 + // In non-thinking mode the channel is dropped entirely.
400 + if thinking {
401 + write_open_tag(out, "think", &[])?;
402 + if !reasoning.trim().is_empty() {
403 + out.ordinary(&reasoning)?;
404 + }
405 + write_close_tag(out, "think")?;
406 + }
407 +
408 + write_open_tag(out, "response", &[])?;
409 + out.ordinary(&response)?;
410 + write_close_tag(out, "response")?;
411 +
412 + if !tool_calls.is_empty() {
413 + write_open_tag(out, "tools", &[])?;
414 + for (index, tool_call) in tool_calls.into_iter().enumerate() {
415 + write_assistant_tool_call(out, tool_call, index + 1)?;
416 + }
417 + write_close_tag(out, "tools")?;
418 + }
419 +
420 + write_close_tag(out, "message")?;
421 + out.control(END_OF_MSG)
422 +}
423 +
424 +fn write_assistant_tool_call(
425 + out: &mut K3TokenWriter<'_>,
426 + tool_call: &AssistantToolCall,
427 + index: usize,
428 +) -> Result<()> {
429 + let index_str = index.to_string();
430 + write_open_tag(
431 + out,
432 + "call",
433 + &[
434 + ("tool", tool_call.name.as_str()),
435 + ("index", index_str.as_str()),
436 + ],
437 + )?;
438 +
439 + let (args, json_block) = normalize_tool_arguments(&tool_call.arguments)?;
440 + if let Some(raw) = json_block {
441 + write_open_tag(out, "json", &[("type", "object")])?;
442 + out.ordinary(&raw)?;
443 + write_close_tag(out, "json")?;
444 + } else {
445 + for (key, value) in args {
446 + let typ = xtml_type(&value);
447 + write_open_tag(out, "argument", &[("key", key.as_str()), ("type", typ)])?;
448 + out.ordinary(&xtml_value(&value))?;
449 + write_close_tag(out, "argument")?;
450 + }
451 + }
452 +
453 + write_close_tag(out, "call")
454 +}
455 +
456 +fn write_content(out: &mut K3TokenWriter<'_>, content: &ChatContent) -> Result<()> {
457 + match content {
458 + ChatContent::Text(text) => write_text_with_images(out, text),
459 + ChatContent::Parts(parts) => {
460 + for part in parts {
461 + match part {
462 + ChatContentPart::Text { text } => write_text_with_images(out, text)?,
463 + ChatContentPart::ImageUrl { .. } => out.control(IMAGE_PLACEHOLDER)?,
464 + ChatContentPart::VideoUrl { .. } => {
465 + return Err(Error::UnsupportedMultimodalContent("video_url"));
466 + }
467 + ChatContentPart::InputAudio { .. } => {
468 + return Err(Error::UnsupportedMultimodalContent("input_audio"));
469 + }
470 + ChatContentPart::AudioUrl { .. } => {
471 + return Err(Error::UnsupportedMultimodalContent("audio_url"));
472 + }
473 + }
474 + }
475 + Ok(())
476 + }
477 + }
478 +}
479 +
480 +fn write_text_with_images(out: &mut K3TokenWriter<'_>, text: &str) -> Result<()> {
481 + // Placeholder expansion is left as the literal K3 image token; multimodal
482 + // preprocessing can replace it once image prompts are known.
483 + out.ordinary(text)
484 +}
485 +
486 +fn write_response_format(out: &mut K3TokenWriter<'_>, request: &ChatRequest) -> Result<()> {
487 + let Some(rf) = request.chat_options.response_format.as_ref() else {
488 + return Ok(());
489 + };
490 +
491 + let rf_type = rf.get("type").and_then(Value::as_str).or_else(|| rf.as_str()).unwrap_or("");
492 +
493 + match rf_type {
494 + "json_object" => {
495 + write_internal_system(
496 + out,
497 + "response-format",
498 + "The system is invoked with `response_format=json_object`.\n\
499 + Your response must be raw JSON data without markdown code \
500 + blocks (```json) or any additional formatting.",
501 + )?;
502 + }
503 + "json_schema" => {
504 + let schema = extract_response_schema(rf);
505 + let schema_json = compact_json(&sort_json(&schema.unwrap_or(Value::Null)))?;
506 + write_internal_system(
507 + out,
508 + "response-format",
509 + &format!(
510 + "The system is invoked with `response_format=json_schema`.\n\
511 + Your response must be raw JSON data without markdown code \
512 + blocks (```json) or any additional formatting.\n\
513 + The JSON data must match the following schema:\n\
514 + ```json\n\
515 + {schema_json}\n\
516 + ```"
517 + ),
518 + )?;
519 + }
520 + _ => {}
521 + }
522 + Ok(())
523 +}
524 +
525 +fn extract_response_schema(response_format: &Value) -> Option<Value> {
526 + let json_schema = response_format.get("json_schema")?;
527 + if let Some(schema) = json_schema.get("schema") {
528 + return Some(schema.clone());
529 + }
530 + if let Some(schema) = json_schema.get("json_schema") {
531 + return Some(schema.clone());
532 + }
533 + Some(json_schema.clone())
534 +}
535 +
536 +fn normalize_tool_arguments(arguments: &str) -> Result<(Map<String, Value>, Option<String>)> {
537 + let trimmed = arguments.trim();
538 + if trimmed.is_empty() {
539 + return Ok((Map::new(), None));
540 + }
541 + match serde_json::from_str::<Value>(trimmed) {
542 + Ok(Value::Object(map)) => Ok((map, None)),
543 + Ok(_) => Err(Error::ChatTemplate(
544 + "Kimi K3 tool call arguments must be a JSON object".to_string(),
545 + )),
546 + Err(_) => Ok((Map::new(), Some(arguments.to_string()))),
547 + }
548 +}
549 +
550 +fn xtml_type(value: &Value) -> &'static str {
551 + match value {
552 + Value::Bool(_) => "boolean",
553 + Value::Null => "null",
554 + Value::Number(_) => "number",
555 + Value::String(_) => "string",
556 + Value::Object(_) => "object",
557 + Value::Array(_) => "array",
558 + }
559 +}
560 +
561 +fn xtml_value(value: &Value) -> String {
562 + match value {
563 + Value::String(text) => text.clone(),
564 + other => compact_json(other).unwrap_or_else(|_| other.to_string()),
565 + }
566 +}
567 +
568 +fn write_open_tag(out: &mut K3TokenWriter<'_>, tag: &str, attrs: &[(&str, &str)]) -> Result<()> {
569 + out.control(OPEN)?;
570 + out.ordinary(tag)?;
571 + for (key, value) in attrs {
572 + out.ordinary(&format!(" {key}"))?;
573 + out.ordinary("=\"")?;
574 + out.ordinary(&escape_attr_value(value))?;
575 + out.ordinary("\"")?;
576 + }
577 + out.control(SEP)
578 +}
579 +
580 +fn write_close_tag(out: &mut K3TokenWriter<'_>, tag: &str) -> Result<()> {
581 + out.control(CLOSE)?;
582 + out.ordinary(tag)?;
583 + out.control(SEP)
584 +}
585 +
586 +fn escape_attr_value(value: &str) -> String {
587 + value.replace('&', "&amp;").replace('"', "&quot;")
588 +}
589 +
590 +fn compact_json(value: &Value) -> Result<String> {
591 + serde_json::to_string(value).map_err(|error| Error::ChatTemplate(error.to_string()))
592 +}
593 +
594 +fn sort_json(value: &Value) -> Value {
595 + match value {
596 + Value::Array(items) => Value::Array(items.iter().map(sort_json).collect()),
597 + Value::Object(map) => {
598 + let mut sorted = Map::new();
599 + let mut keys = map.keys().collect::<Vec<_>>();
600 + keys.sort();
601 + for key in keys {
602 + sorted.insert(key.clone(), sort_json(&map[key]));
603 + }
604 + Value::Object(sorted)
605 + }
606 + _ => value.clone(),
607 + }
608 +}
rust/src/chat/src/renderer/kimi_k3/fixtures/controls_thinking_off_input.json new
+15
@@ -0,0 +1,15 @@
1 +{
2 + "messages": [
3 + {
4 + "role": "user",
5 + "content": "json pls"
6 + }
7 + ],
8 + "add_generation_prompt": true,
9 + "response_format": {
10 + "type": "json_object"
11 + },
12 + "template_kwargs": {
13 + "thinking": false
14 + }
15 +}
rust/src/chat/src/renderer/kimi_k3/fixtures/controls_thinking_off_output.txt new
+2
@@ -0,0 +1,2 @@
1 +<|open|>message role="user"<|sep|>json pls<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="response-format"<|sep|>The system is invoked with `response_format=json_object`.
2 +Your response must be raw JSON data without markdown code blocks (```json) or any additional formatting.<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>response<|sep|>
\ No newline at end of file
rust/src/chat/src/renderer/kimi_k3/fixtures/dynamic_system_tool_declare_input.json new
+62
@@ -0,0 +1,62 @@
1 +{
2 + "tools": [
3 + {
4 + "type": "function",
5 + "function": {
6 + "name": "get_weather",
7 + "description": "weather",
8 + "parameters": {
9 + "type": "object",
10 + "properties": {
11 + "city": {
12 + "type": "string"
13 + },
14 + "days": {
15 + "type": "number"
16 + }
17 + },
18 + "required": [
19 + "city"
20 + ]
21 + }
22 + }
23 + }
24 + ],
25 + "messages": [
26 + {
27 + "role": "user",
28 + "content": "hi"
29 + },
30 + {
31 + "role": "developer",
32 + "tools": [
33 + {
34 + "type": "function",
35 + "function": {
36 + "name": "calc",
37 + "description": "calculator",
38 + "parameters": {
39 + "type": "object",
40 + "properties": {
41 + "x": {
42 + "type": "number"
43 + }
44 + },
45 + "required": [
46 + "x"
47 + ]
48 + }
49 + }
50 + }
51 + ]
52 + },
53 + {
54 + "role": "user",
55 + "content": "use calc"
56 + }
57 + ],
58 + "add_generation_prompt": true,
59 + "template_kwargs": {
60 + "thinking": true
61 + }
62 +}
rust/src/chat/src/renderer/kimi_k3/fixtures/dynamic_system_tool_declare_output.txt new
+14
@@ -0,0 +1,14 @@
1 +<|open|>message role="system" type="tool-declare"<|sep|># Tools
2 +Here are the available tools, described in JSONSchema.
3 +
4 +```json
5 +[{"function":{"description":"weather","name":"get_weather","parameters":{"properties":{"city":{"type":"string"},"days":{"type":"number"}},"required":["city"],"type":"object"}},"type":"function"}]
6 +```<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
7 +Now the system is invoked with `thinking_effort=max`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>hi<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="tool-declare"<|sep|>## New Tools Available
8 +The system dynamically extends the toolset via lazy-loading.
9 +You have access to all existing and extended tools.
10 +Here are the specs for the extended tools.
11 +
12 +```json
13 +[{"function":{"description":"calculator","name":"calc","parameters":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}},"type":"function"}]
14 +```<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>use calc<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>
\ No newline at end of file
rust/src/chat/src/renderer/kimi_k3/fixtures/history_preserve_and_image_input.json new
+31
@@ -0,0 +1,31 @@
1 +{
2 + "messages": [
3 + {
4 + "role": "user",
5 + "content": [
6 + {
7 + "type": "text",
8 + "text": "see this"
9 + },
10 + {
11 + "type": "image_url",
12 + "image_url": "https://example.com/a.png"
13 + }
14 + ]
15 + },
16 + {
17 + "role": "assistant",
18 + "content": "old answer",
19 + "reasoning_content": "old reasoning"
20 + },
21 + {
22 + "role": "user",
23 + "content": "continue"
24 + }
25 + ],
26 + "add_generation_prompt": true,
27 + "template_kwargs": {
28 + "thinking": true,
29 + "thinking_effort": "high"
30 + }
31 +}
rust/src/chat/src/renderer/kimi_k3/fixtures/history_preserve_and_image_output.txt new
+2
@@ -0,0 +1,2 @@
1 +<|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
2 +Now the system is invoked with `thinking_effort=high`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>see this<|media_pad|><|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>old reasoning<|close|>think<|sep|><|open|>response<|sep|>old answer<|close|>response<|sep|><|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>continue<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>
\ No newline at end of file
rust/src/chat/src/renderer/kimi_k3/fixtures/tools_history_and_required_input.json new
+87
@@ -0,0 +1,87 @@
1 +{
2 + "tools": [
3 + {
4 + "type": "function",
5 + "function": {
6 + "name": "get_weather",
7 + "description": "weather",
8 + "parameters": {
9 + "type": "object",
10 + "properties": {
11 + "city": {
12 + "type": "string"
13 + },
14 + "days": {
15 + "type": "number"
16 + }
17 + },
18 + "required": [
19 + "city"
20 + ]
21 + }
22 + }
23 + },
24 + {
25 + "type": "function",
26 + "function": {
27 + "name": "calc",
28 + "description": "calculator",
29 + "parameters": {
30 + "type": "object",
31 + "properties": {
32 + "x": {
33 + "type": "number"
34 + }
35 + },
36 + "required": [
37 + "x"
38 + ]
39 + }
40 + }
41 + }
42 + ],
43 + "messages": [
44 + {
45 + "role": "user",
46 + "content": "Hangzhou weather and calc"
47 + },
48 + {
49 + "role": "assistant",
50 + "content": "I'll check.",
51 + "reasoning_content": "Need tools.",
52 + "tool_calls": [
53 + {
54 + "id": "get_weather:0",
55 + "type": "function",
56 + "function": {
57 + "name": "get_weather",
58 + "arguments": "{\"city\":\"Hangzhou\",\"days\":1}"
59 + }
60 + },
61 + {
62 + "id": "calc:1",
63 + "type": "function",
64 + "function": {
65 + "name": "calc",
66 + "arguments": "{\"x\":2}"
67 + }
68 + }
69 + ]
70 + },
71 + {
72 + "role": "tool",
73 + "tool_call_id": "calc:1",
74 + "content": "2"
75 + },
76 + {
77 + "role": "tool",
78 + "tool_call_id": "get_weather:0",
79 + "content": "{\"city\":\"Hangzhou\",\"condition\":\"rain\"}"
80 + }
81 + ],
82 + "add_generation_prompt": true,
83 + "tool_choice": "required",
84 + "template_kwargs": {
85 + "thinking": true
86 + }
87 +}
rust/src/chat/src/renderer/kimi_k3/fixtures/tools_history_and_required_output.txt new
+8
@@ -0,0 +1,8 @@
1 +<|open|>message role="system" type="tool-declare"<|sep|># Tools
2 +Here are the available tools, described in JSONSchema.
3 +
4 +```json
5 +[{"function":{"description":"weather","name":"get_weather","parameters":{"properties":{"city":{"type":"string"},"days":{"type":"number"}},"required":["city"],"type":"object"}},"type":"function"},{"function":{"description":"calculator","name":"calc","parameters":{"properties":{"x":{"type":"number"}},"required":["x"],"type":"object"}},"type":"function"}]
6 +```<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
7 +Now the system is invoked with `thinking_effort=max`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>Hangzhou weather and calc<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>Need tools.<|close|>think<|sep|><|open|>response<|sep|>I'll check.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool="get_weather" index="1"<|sep|><|open|>argument key="city" type="string"<|sep|>Hangzhou<|close|>argument<|sep|><|open|>argument key="days" type="number"<|sep|>1<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool="calc" index="2"<|sep|><|open|>argument key="x" type="number"<|sep|>2<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|><|end_of_msg|><|open|>message role="tool" tool="get_weather" index="1"<|sep|>{"city":"Hangzhou","condition":"rain"}<|close|>message<|sep|><|end_of_msg|><|open|>message role="tool" tool="calc" index="2"<|sep|>2<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="tool-choice"<|sep|>The system is invoked with `tool_choice=required`.
8 +You MUST call tools in the next message.<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>
\ No newline at end of file
rust/src/chat/src/renderer/kimi_k3/mod.rs new
+39
@@ -0,0 +1,39 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +//! Native Kimi K3 XTML chat renderer.
5 +
6 +mod encoding;
7 +#[cfg(test)]
8 +mod tests;
9 +
10 +use vllm_text::Prompt;
11 +use vllm_text::tokenizer::DynTokenizer;
12 +
13 +use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
14 +use crate::Result;
15 +use crate::request::ChatRequest;
16 +
17 +/// Dedicated Kimi K3 XTML renderer.
18 +#[derive(Clone)]
19 +pub struct KimiK3ChatRenderer {
20 + tokenizer: DynTokenizer,
21 +}
22 +
23 +impl KimiK3ChatRenderer {
24 + /// Create a Kimi K3 renderer.
25 + pub fn new(tokenizer: DynTokenizer) -> Self {
26 + Self { tokenizer }
27 + }
28 +}
29 +
30 +impl ChatRenderer for KimiK3ChatRenderer {
31 + fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
32 + request.validate()?;
33 +
34 + Ok(RenderedPrompt {
35 + prompt: Prompt::TokenIds(encoding::render_request(request, self.tokenizer.as_ref())?),
36 + effective_template_kwargs: request_template_kwargs(request),
37 + })
38 + }
39 +}
rust/src/chat/src/renderer/kimi_k3/tests.rs new
+267
@@ -0,0 +1,267 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +//! Golden fixtures generated from HF remote-code `encoding_k3.py`.
5 +
6 +use std::path::PathBuf;
7 +use std::sync::Arc;
8 +
9 +use expect_test::{expect, expect_file};
10 +use serde_json::json;
11 +use vllm_text::Prompt;
12 +use vllm_text::tokenizer::DynTokenizer;
13 +use vllm_tokenizer::Tokenizer;
14 +use vllm_tokenizer::test_utils::TestTokenizer;
15 +
16 +use super::KimiK3ChatRenderer;
17 +use crate::AssistantContentBlock;
18 +use crate::ChatRenderer;
19 +use crate::renderer::kimi_k3::encoding::{CLOSE, END_OF_MSG, IMAGE_PLACEHOLDER, OPEN, SEP};
20 +use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
21 +use crate::request::{ChatContentPart, ChatMessage, GenerationPromptMode, ReasoningEffort};
22 +
23 +const OPEN_ID: u32 = 256;
24 +const CLOSE_ID: u32 = 257;
25 +const SEP_ID: u32 = 258;
26 +const END_OF_MSG_ID: u32 = 259;
27 +const MEDIA_ID: u32 = 260;
28 +
29 +fn test_tokenizer() -> TestTokenizer {
30 + TestTokenizer::new()
31 + .with_special_token(OPEN, OPEN_ID)
32 + .with_special_token(CLOSE, CLOSE_ID)
33 + .with_special_token(SEP, SEP_ID)
34 + .with_special_token(END_OF_MSG, END_OF_MSG_ID)
35 + .with_special_token(IMAGE_PLACEHOLDER, MEDIA_ID)
36 +}
37 +
38 +fn render_token_ids(request: &crate::request::ChatRequest, tokenizer: DynTokenizer) -> Vec<u32> {
39 + let prompt = KimiK3ChatRenderer::new(tokenizer).render(request).unwrap().prompt;
40 + let Prompt::TokenIds(token_ids) = prompt else {
41 + panic!("kimi k3 renderer should return token IDs")
42 + };
43 + token_ids
44 +}
45 +
46 +fn render_request(request: &crate::request::ChatRequest) -> String {
47 + let tokenizer: DynTokenizer = Arc::new(test_tokenizer());
48 + let token_ids = render_token_ids(request, tokenizer.clone());
49 + tokenizer.decode(&token_ids, false).unwrap()
50 +}
51 +
52 +fn fixture_path(name: &str) -> PathBuf {
53 + PathBuf::from(env!("CARGO_MANIFEST_DIR"))
54 + .join("src/renderer/kimi_k3/fixtures")
55 + .join(name)
56 +}
57 +
58 +fn kimi_k3_fixture_options() -> FixtureRequestOptions {
59 + FixtureRequestOptions {
60 + // Fixture JSON owns thinking via `template_kwargs`.
61 + enable_thinking: None,
62 + no_generation_prompt_when_last_assistant: false,
63 + }
64 +}
65 +
66 +fn assert_golden(name: &str) {
67 + let input_name = format!("{name}_input.json");
68 + let request = fixture_chat_request(&fixture_path(&input_name), kimi_k3_fixture_options());
69 + let rendered = render_request(&request);
70 + expect_file![format!("fixtures/{name}_output.txt")].assert_eq(&rendered);
71 +}
72 +
73 +#[test]
74 +fn golden_history_preserve_and_image() {
75 + assert_golden("history_preserve_and_image");
76 +}
77 +
78 +#[test]
79 +fn golden_tools_history_and_required() {
80 + assert_golden("tools_history_and_required");
81 +}
82 +
83 +#[test]
84 +fn golden_controls_thinking_off() {
85 + assert_golden("controls_thinking_off");
86 +}
87 +
88 +#[test]
89 +fn golden_dynamic_system_tool_declare() {
90 + assert_golden("dynamic_system_tool_declare");
91 +}
92 +
93 +#[test]
94 +fn token_writer_protects_literal_control_and_media_markers() {
95 + let tokenizer = Arc::new(test_tokenizer());
96 + let user_text = format!("literal {OPEN} and {}", super::encoding::IMAGE_PLACEHOLDER);
97 + let mut request = crate::request::ChatRequest::for_test();
98 + request.messages = vec![ChatMessage::user(vec![
99 + ChatContentPart::text(user_text),
100 + ChatContentPart::image_url("data:image/png;base64,test"),
101 + ])];
102 + request
103 + .chat_options
104 + .template_kwargs
105 + .insert("thinking".to_string(), json!(false));
106 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
107 +
108 + let token_ids = render_token_ids(&request, tokenizer.clone());
109 +
110 + assert_eq!(
111 + token_ids.iter().filter(|&&token_id| token_id == OPEN_ID).count(),
112 + 1
113 + );
114 + assert_eq!(
115 + token_ids.iter().filter(|&&token_id| token_id == MEDIA_ID).count(),
116 + 1
117 + );
118 +
119 + let flattened = tokenizer.decode(&token_ids, false).unwrap();
120 + let flattened_ids = tokenizer.encode(&flattened, false).unwrap();
121 + assert_eq!(
122 + flattened_ids.iter().filter(|&&token_id| token_id == OPEN_ID).count(),
123 + 2
124 + );
125 + assert_eq!(
126 + flattened_ids.iter().filter(|&&token_id| token_id == MEDIA_ID).count(),
127 + 2
128 + );
129 +}
130 +
131 +#[test]
132 +fn thinking_history_renders_empty_think_channel() {
133 + let mut request = crate::request::ChatRequest::for_test();
134 + request.messages = vec![
135 + ChatMessage::user("question"),
136 + ChatMessage::assistant_text("answer"),
137 + ChatMessage::user("follow-up"),
138 + ];
139 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
140 +
141 + let rendered = render_request(&request);
142 +
143 + assert!(rendered.contains(
144 + "<|open|>message role=\"assistant\"<|sep|>\
145 + <|open|>think<|sep|><|close|>think<|sep|>\
146 + <|open|>response<|sep|>answer<|close|>response<|sep|>"
147 + ));
148 +}
149 +
150 +#[test]
151 +fn non_thinking_history_omits_reasoning_channel() {
152 + let mut request = crate::request::ChatRequest::for_test();
153 + request.messages = vec![
154 + ChatMessage::user("question"),
155 + ChatMessage::assistant_blocks(vec![
156 + AssistantContentBlock::Reasoning {
157 + text: "hidden reasoning".to_string(),
158 + },
159 + AssistantContentBlock::Text {
160 + text: "answer".to_string(),
161 + },
162 + ]),
163 + ChatMessage::user("follow-up"),
164 + ];
165 + request
166 + .chat_options
167 + .template_kwargs
168 + .insert("thinking".to_string(), json!(false));
169 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
170 +
171 + let rendered = render_request(&request);
172 +
173 + assert!(!rendered.contains("hidden reasoning"));
174 + assert!(!rendered.contains("<|open|>think<|sep|>"));
175 + assert!(rendered.contains(
176 + "<|open|>message role=\"assistant\"<|sep|>\
177 + <|open|>response<|sep|>answer<|close|>response<|sep|>"
178 + ));
179 +}
180 +
181 +#[test]
182 +fn defaults_thinking_effort_to_max() {
183 + let rendered = render_request(&crate::request::ChatRequest::for_test());
184 +
185 + expect![[r#"<|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
186 +Now the system is invoked with `thinking_effort=max`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>test<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>"#]]
187 + .assert_eq(&rendered);
188 +}
189 +
190 +#[test]
191 +fn translates_standard_thinking_kwargs() {
192 + let mut request = crate::request::ChatRequest::for_test();
193 + request
194 + .chat_options
195 + .template_kwargs
196 + .insert("enable_thinking".to_string(), json!(true));
197 + request
198 + .chat_options
199 + .template_kwargs
200 + .insert("reasoning_effort".to_string(), json!("high"));
201 +
202 + let rendered = render_request(&request);
203 +
204 + assert!(rendered.contains("thinking_effort=high"));
205 + assert!(rendered.ends_with("<|open|>think<|sep|>"));
206 +}
207 +
208 +#[test]
209 +fn native_k3_kwargs_take_precedence() {
210 + let mut request = crate::request::ChatRequest::for_test();
211 + request.chat_options.template_kwargs.extend([
212 + ("thinking".to_string(), json!(true)),
213 + ("enable_thinking".to_string(), json!(false)),
214 + ("thinking_effort".to_string(), json!("low")),
215 + ("reasoning_effort".to_string(), json!("high")),
216 + ]);
217 +
218 + let rendered = render_request(&request);
219 +
220 + assert!(rendered.contains("thinking_effort=low"));
221 + assert!(!rendered.contains("thinking_effort=high"));
222 +}
223 +
224 +#[test]
225 +fn standard_none_disables_thinking() {
226 + let mut request = crate::request::ChatRequest::for_test();
227 + request.chat_options.template_kwargs.extend([
228 + ("enable_thinking".to_string(), json!(false)),
229 + ("reasoning_effort".to_string(), json!("none")),
230 + ]);
231 +
232 + let rendered = render_request(&request);
233 +
234 + assert!(!rendered.contains("type=\"thinking-effort\""));
235 + assert!(rendered.ends_with("<|open|>response<|sep|>"));
236 +}
237 +
238 +#[test]
239 +fn typed_none_disables_thinking() {
240 + let mut request = crate::request::ChatRequest::for_test();
241 + request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
242 +
243 + let rendered = render_request(&request);
244 +
245 + assert!(!rendered.contains("type=\"thinking-effort\""));
246 + assert!(rendered.ends_with("<|open|>response<|sep|>"));
247 +}
248 +
249 +#[test]
250 +fn rejects_removed_medium_thinking_effort() {
251 + let mut request = crate::request::ChatRequest::for_test();
252 + request
253 + .chat_options
254 + .template_kwargs
255 + .insert("thinking_effort".to_string(), json!("medium"));
256 +
257 + let error = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()))
258 + .render(&request)
259 + .unwrap_err();
260 +
261 + expect![[r#"
262 + ChatTemplate(
263 + "unsupported thinking_effort=\"medium\"; supported values are `low`, `high`, and `max`",
264 + )
265 + "#]]
266 + .assert_debug_eq(&error);
267 +}
rust/src/chat/src/renderer/mod.rs
+2
@@ -15,6 +15,7 @@ pub mod deepseek_v4;
15 pub mod harmony;
16 pub mod hf;
17 mod inkling;
18 +mod kimi_k3;
19 mod selection;
20 #[cfg(test)]
21 mod test_utils;
@@ -23,6 +24,7 @@ pub use deepseek_v4::DeepSeekV4ChatRenderer;
24 pub use deepseek_v32::DeepSeekV32ChatRenderer;
25 pub use harmony::HarmonyChatRenderer;
26 pub use inkling::InklingChatRenderer;
27 +pub use kimi_k3::KimiK3ChatRenderer;
28 pub use selection::RendererSelection;
29
30 /// Rendered chat prompt submitted to the text backend.
rust/src/chat/src/renderer/selection.rs
+8 -1
@@ -26,6 +26,8 @@ pub enum RendererSelection {
26 Harmony,
27 /// Force the Inkling native token renderer.
28 Inkling,
29 + /// Force the Kimi K3 XTML renderer.
30 + KimiK3,
31 }
32
33 impl RendererSelection {
@@ -37,6 +39,7 @@ impl RendererSelection {
39 pub const HF_LITERAL: &str = "hf";
40 pub const INKLING_LITERAL: &str = "inkling";
41 pub const INKLING_MODEL_TYPE: &str = "inkling_mm_model";
42 + pub const KIMI_K3_LITERAL: &str = "kimi_k3";
43
44 /// Resolve the renderer selection using the given model type string, if
45 /// it's `Auto`.
@@ -47,6 +50,7 @@ impl RendererSelection {
50 Self::DEEPSEEK_V4_LITERAL => Self::DeepSeekV4,
51 Self::GPT_OSS_MODEL_TYPE => Self::Harmony,
52 Self::INKLING_MODEL_TYPE => Self::Inkling,
53 + Self::KIMI_K3_LITERAL => Self::KimiK3,
54 _ => Self::Hf,
55 },
56 selection => selection,
@@ -70,6 +74,8 @@ impl FromStr for RendererSelection {
74 Ok(Self::Harmony)
75 } else if value.eq_ignore_ascii_case(Self::INKLING_LITERAL) {
76 Ok(Self::Inkling)
77 + } else if value.eq_ignore_ascii_case(Self::KIMI_K3_LITERAL) {
78 + Ok(Self::KimiK3)
79 } else {
80 Err(format!(
81 "unknown renderer `{value}` (expected one of: {})",
@@ -88,6 +94,7 @@ impl fmt::Display for RendererSelection {
94 Self::DeepSeekV4 => f.write_str(Self::DEEPSEEK_V4_LITERAL),
95 Self::Harmony => f.write_str(Self::HARMONY_LITERAL),
96 Self::Inkling => f.write_str(Self::INKLING_LITERAL),
97 + Self::KimiK3 => f.write_str(Self::KIMI_K3_LITERAL),
98 }
99 }
100 }
@@ -114,7 +121,7 @@ mod tests {
121 fn renderer_selection_expected_error_message() {
122 let err = RendererSelection::from_str("unknown").unwrap_err();
123 expect_test::expect![
117 - "unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling)"
124 + "unknown renderer `unknown` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3)"
125 ]
126 .assert_eq(&err);
127 }
rust/src/chat/src/renderer/test_utils.rs
+41 -11
@@ -1,6 +1,7 @@
1 // SPDX-License-Identifier: Apache-2.0
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +use std::collections::HashMap;
5 use std::fs;
6 use std::path::Path;
7
@@ -16,8 +17,9 @@ use crate::request::{
17 /// Options for constructing a [`ChatRequest`] from a fixture file.
18 #[derive(Debug, Clone, Copy)]
19 pub(crate) struct FixtureRequestOptions {
19 - /// Whether to set the template kwarg `[enable_]thinking=true`.
20 - pub enable_thinking: bool,
20 + /// Optional thinking toggle applied only when the fixture does not already
21 + /// set `thinking` / `enable_thinking` in [`FixtureRequest::template_kwargs`].
22 + pub enable_thinking: Option<bool>,
23 /// Whether fixtures ending in an assistant message should omit the
24 /// trailing generation prompt.
25 pub no_generation_prompt_when_last_assistant: bool,
@@ -46,6 +48,15 @@ pub(crate) struct FixtureRequest {
48 messages: Vec<FixtureMessage>,
49 add_generation_prompt: Option<bool>,
50 reasoning_effort: Option<ReasoningEffort>,
51 + /// Standard response format passed to model-specific renderers.
52 + #[serde(default)]
53 + response_format: Option<Value>,
54 + /// Extra chat-template kwargs (thinking, preserve_thinking, …).
55 + #[serde(default)]
56 + template_kwargs: HashMap<String, Value>,
57 + /// When omitted, defaults to `auto` if tools are present, otherwise `none`.
58 + #[serde(default)]
59 + tool_choice: Option<ChatToolChoice>,
60 }
61
62 impl FixtureFile {
@@ -57,6 +68,9 @@ impl FixtureFile {
68 messages,
69 add_generation_prompt: None,
70 reasoning_effort: None,
71 + response_format: None,
72 + template_kwargs: HashMap::new(),
73 + tool_choice: None,
74 },
75 }
76 }
@@ -69,6 +83,7 @@ pub(crate) enum FixtureMessage {
83 content: FixtureContent,
84 },
85 Developer {
86 + #[serde(default)]
87 content: FixtureContent,
88 #[serde(default)]
89 tools: Vec<FixtureTool>,
@@ -98,6 +113,12 @@ pub(crate) enum FixtureContent {
113 Parts(Vec<FixtureContentPart>),
114 }
115
116 +impl Default for FixtureContent {
117 + fn default() -> Self {
118 + Self::Text(String::new())
119 + }
120 +}
121 +
122 #[derive(Debug, Deserialize)]
123 #[serde(tag = "type", rename_all = "snake_case")]
124 pub(crate) enum FixtureContentPart {
@@ -136,6 +157,13 @@ struct FixtureToolCallFunction {
157
158 impl FixtureRequest {
159 fn into_chat_request(self, options: FixtureRequestOptions) -> ChatRequest {
160 + let tools = to_chat_tools(&self.tools);
161 + let tool_choice = self.tool_choice.unwrap_or(if tools.is_empty() {
162 + ChatToolChoice::None
163 + } else {
164 + ChatToolChoice::Auto
165 + });
166 +
167 let mut request = ChatRequest {
168 request_id: "renderer-fixture".to_string(),
169 messages: self
@@ -144,12 +172,8 @@ impl FixtureRequest {
172 .enumerate()
173 .map(|(index, message)| fixture_message_to_chat_message(index, message))
174 .collect(),
147 - tools: to_chat_tools(&self.tools),
148 - tool_choice: if self.tools.is_empty() {
149 - ChatToolChoice::None
150 - } else {
151 - ChatToolChoice::Auto
152 - },
175 + tools,
176 + tool_choice,
177 ..ChatRequest::for_test()
178 };
179
@@ -162,9 +186,15 @@ impl FixtureRequest {
186 request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
187 }
188 request.chat_options.reasoning_effort = self.reasoning_effort;
165 - if options.enable_thinking {
166 - for key in ["thinking", "enable_thinking"] {
167 - request.chat_options.template_kwargs.insert(key.to_string(), Value::Bool(true));
189 + request.chat_options.response_format = self.response_format;
190 + request.chat_options.template_kwargs.extend(self.template_kwargs);
191 +
192 + // Options supply a default thinking toggle only when the fixture did not.
193 + if let Some(thinking) = options.enable_thinking {
194 + let kwargs = &mut request.chat_options.template_kwargs;
195 + if !kwargs.contains_key("thinking") && !kwargs.contains_key("enable_thinking") {
196 + kwargs.insert("thinking".to_string(), Value::Bool(thinking));
197 + kwargs.insert("enable_thinking".to_string(), Value::Bool(thinking));
198 }
199 }
200
rust/src/chat/src/request.rs
+5
@@ -397,6 +397,10 @@ pub struct ChatOptions {
397 /// Effort level exposed to chat templates for reasoning models.
398 pub reasoning_effort: Option<ReasoningEffort>,
399
400 + /// Standard response format available to model-specific renderers.
401 + #[serde(default)]
402 + pub response_format: Option<Value>,
403 +
404 /// Additional keyword arguments exposed to the chat template.
405 pub template_kwargs: HashMap<String, Value>,
406 }
@@ -407,6 +411,7 @@ impl Default for ChatOptions {
411 generation_prompt_mode: GenerationPromptMode::StartNewAssistant,
412 chat_template: None,
413 reasoning_effort: None,
414 + response_format: None,
415 template_kwargs: HashMap::new(),
416 }
417 }
rust/src/chat/tests/roundtrip.rs
+35 -7
@@ -212,6 +212,23 @@ impl RoundtripCase {
212 }
213 }
214
215 + /// Kimi K3 XTML tool/reasoning channels (native renderer + unified parser).
216 + ///
217 + /// Needs HF tokenizer files under `HF_HOME` (`tiktoken.model` +
218 + /// `tokenizer_config.json`). Weights are not required for this text-level
219 + /// roundtrip.
220 + fn kimi_k3() -> Self {
221 + Self {
222 + model_id: "moonshotai/Kimi-K3",
223 + assistant_stop_suffix: "<|end_of_msg|>",
224 + tool_call_parser: ParserSelection::Auto,
225 + reasoning_parser: ParserSelection::Auto,
226 + thinking_behavior: ThinkingBehavior::Toggleable { default: true },
227 + json_fmt: compact_json_fmt(),
228 + sort_json_keys: false,
229 + }
230 + }
231 +
232 /// SeedOSS with `<seed:think>` / `</seed:think>` reasoning tags.
233 fn seed_oss() -> Self {
234 Self {
@@ -255,7 +272,7 @@ impl RoundtripCase {
272 fn gpt_oss() -> Self {
273 Self {
274 model_id: "openai/gpt-oss-20b",
258 - assistant_stop_suffix: "", // not applicable for token-id cases
275 + assistant_stop_suffix: "",
276 tool_call_parser: ParserSelection::Auto,
277 reasoning_parser: ParserSelection::Auto,
278 thinking_behavior: ThinkingBehavior::Always { value: true },
@@ -268,7 +285,7 @@ impl RoundtripCase {
285 fn inkling() -> Self {
286 Self {
287 model_id: "thinkingmachines/Inkling",
271 - assistant_stop_suffix: "",
288 + assistant_stop_suffix: "<|content_model_end_sampling|>",
289 tool_call_parser: ParserSelection::Auto,
290 reasoning_parser: ParserSelection::Auto,
291 thinking_behavior: ThinkingBehavior::Always { value: true },
@@ -312,6 +329,8 @@ roundtrip_tests! {
329 nemotron_v3 => [reasoning_and_content],
330 gemma4 => [tool_call_mix], // Gemma4 strips reasoning in history if there's no tool call
331 kimi_k25 => [tool_call_mix], // Kimi K2.5 strips reasoning in history
332 + // K3 drops plain-assistant reasoning in history; tool-call turns keep it.
333 + kimi_k3 => [tool_call_mix],
334 gpt_oss => [tool_call_mix], // Harmony strips reasoning in history if there's no tool call
335 inkling => [reasoning_and_content, tool_call_mix],
336 }
@@ -670,14 +689,23 @@ fn decoded_completion_stream(
689 .collect()
690 }
691 Prompt::TokenIds(token_ids) => {
673 - ensure!(
674 - assistant_stop_suffix.is_empty(),
675 - "token-id roundtrip cases do not support text stop suffixes"
676 - );
692 + let body = if assistant_stop_suffix.is_empty() {
693 + token_ids.as_slice()
694 + } else {
695 + let stop_token_ids = tokenizer
696 + .encode(assistant_stop_suffix, false)
697 + .context("failed to encode token-id completion stop suffix")?;
698 + token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| {
699 + format!(
700 + "token-id completion did not end with {:?}: {:?}",
701 + assistant_stop_suffix, token_ids
702 + )
703 + })?
704 + };
705 incremental_decode_chunks(
706 tokenizer,
707 &prompt_token_ids,
680 - token_ids,
708 + body,
709 TOKEN_COMPLETION_CHUNK_TOKENS,
710 )?
711 }
rust/src/cmd/src/cli/tests.rs
+1 -1
@@ -666,7 +666,7 @@ fn serve_args_reject_unknown_renderer_value() {
666 .unwrap_err();
667
668 expect![[r#"
669 - error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling)
669 + error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony, inkling, kimi_k3)
670
671 For more information, try '--help'.
672 "#]]
rust/src/parser/src/unified/kimi_k3.rs new
+1149
@@ -0,0 +1,1149 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +//! Unified parser for the Kimi K3 (XTML) chat format.
5 +//!
6 +//! Original Python implementations:
7 +//! - `vllm/reasoning/kimi_k3_reasoning_parser.py`
8 +//! - `vllm/tool_parsers/kimi_k3_tool_parser.py`
9 +//!
10 +//! K3 wraps one assistant message into XTML channels built from the dedicated
11 +//! special tokens `<|open|>`, `<|close|>`, and `<|sep|>`:
12 +//!
13 +//! ```text
14 +//! <|open|>think<|sep|>reasoning<|close|>think<|sep|>
15 +//! <|open|>response<|sep|>visible answer<|close|>response<|sep|>
16 +//! <|open|>tools<|sep|>
17 +//! <|open|>call tool="get_weather" index="1"<|sep|>
18 +//! <|open|>argument key="city" type="string"<|sep|>Hangzhou<|close|>argument<|sep|>
19 +//! <|close|>call<|sep|>
20 +//! <|close|>tools<|sep|>
21 +//! <|close|>message<|sep|>
22 +//! ```
23 +//!
24 +//! In chat serving the generation prompt ends with `<|open|>think<|sep|>`
25 +//! (thinking) or `<|open|>response<|sep|>` (instruct), so the model output
26 +//! starts *inside* that channel without re-emitting the open tag;
27 +//! [`UnifiedParser::initialize`] detects this from the prompt token IDs.
28 +//!
29 +//! Argument decoding mirrors the renderer's type tagging (inverse encoding):
30 +//! `type="string"` values pass the raw text through, other types are
31 +//! JSON-decoded, and a raw `json` block is passed through unmodified. Attribute
32 +//! values reverse the renderer escaping (`&quot;` before `&amp;`).
33 +//!
34 +//! Known limitation (shared with the Python parser): string argument and
35 +//! response bodies are emitted raw, so a value that literally contains
36 +//! `<|close|>argument<|sep|>` or `<|close|>response<|sep|>` is
37 +//! indistinguishable from a real closing marker.
38 +
39 +mod structural_tag;
40 +
41 +pub use structural_tag::KimiK3StructuralTagBuilder;
42 +
43 +use serde_json::{Map, Value};
44 +use vllm_tokenizer::DynTokenizer;
45 +use winnow::ascii::{multispace0 as ws0, multispace1 as ws1};
46 +use winnow::combinator::{alt, delimited, eof, preceded, repeat, seq, terminated};
47 +use winnow::error::{ContextError, ErrMode, ModalResult, StrContext};
48 +use winnow::prelude::*;
49 +use winnow::stream::Partial;
50 +use winnow::token::{literal, rest, take_till, take_until, take_while};
51 +
52 +use self::structural_tag::KIMI_K3_STRUCTURAL_TAG_BUILDER;
53 +use super::{Result, UnifiedParser, UnifiedParserOutput, token_id};
54 +use crate::tool::{StructuralTagBuilder, Tool, ToolCallDelta};
55 +use crate::unified::parsing_failed;
56 +use crate::utils::{MarkerScanState, parse_buffered_event, safe_text_len_mul, take_until_marker};
57 +
58 +const OPEN: &str = "<|open|>";
59 +const SEP: &str = "<|sep|>";
60 +const END_OF_MSG: &str = "<|end_of_msg|>";
61 +
62 +const THINK_OPEN: &str = "<|open|>think<|sep|>";
63 +const THINK_CLOSE: &str = "<|close|>think<|sep|>";
64 +const RESPONSE_OPEN: &str = "<|open|>response<|sep|>";
65 +const RESPONSE_CLOSE: &str = "<|close|>response<|sep|>";
66 +const TOOLS_OPEN: &str = "<|open|>tools<|sep|>";
67 +const TOOLS_CLOSE: &str = "<|close|>tools<|sep|>";
68 +const MESSAGE_CLOSE: &str = "<|close|>message<|sep|>";
69 +const CALL_OPEN: &str = "<|open|>call";
70 +const CALL_CLOSE: &str = "<|close|>call<|sep|>";
71 +const ARG_OPEN: &str = "<|open|>argument";
72 +const ARG_CLOSE: &str = "<|close|>argument<|sep|>";
73 +const JSON_OPEN: &str = "<|open|>json";
74 +const JSON_CLOSE: &str = "<|close|>json<|sep|>";
75 +
76 +const IDLE_MARKERS: &[&str] = &[
77 + THINK_OPEN,
78 + RESPONSE_OPEN,
79 + TOOLS_OPEN,
80 + MESSAGE_CLOSE,
81 + END_OF_MSG,
82 +];
83 +const REASONING_MARKERS: &[&str] = &[THINK_CLOSE, END_OF_MSG];
84 +const RESPONSE_MARKERS: &[&str] = &[RESPONSE_CLOSE, TOOLS_OPEN, MESSAGE_CLOSE, END_OF_MSG];
85 +const EPILOGUE_MARKERS: &[&str] = &[TOOLS_OPEN, MESSAGE_CLOSE, END_OF_MSG];
86 +const TOOLS_MARKERS: &[&str] = &[CALL_OPEN, TOOLS_CLOSE, MESSAGE_CLOSE, END_OF_MSG];
87 +
88 +/// Channel tags are a couple of text tokens; longer `<|open|>…<|sep|>` spans in
89 +/// the prompt tail (attribute-bearing message opens, message bodies) never name
90 +/// a generation channel.
91 +const MAX_PREFILL_TAG_TOKENS: usize = 8;
92 +
93 +type KimiK3Input<'i> = Partial<&'i str>;
94 +
95 +#[derive(Debug, Clone, PartialEq, Eq)]
96 +enum KimiK3Event {
97 + Text {
98 + len: usize,
99 + },
100 + Reasoning {
101 + len: usize,
102 + },
103 + /// Structural noise consumed without emitting anything.
104 + Skip,
105 + ThinkOpen,
106 + ThinkClose,
107 + ResponseOpen,
108 + ResponseClose,
109 + ToolsOpen,
110 + ToolsClose,
111 + /// The assistant message closed; everything after it is ignored.
112 + MessageEnd,
113 + CallOpen {
114 + name: String,
115 + index: Option<String>,
116 + },
117 + CallComplete {
118 + arguments: String,
119 + },
120 +}
121 +
122 +#[derive(Debug, Clone, Default, PartialEq, Eq)]
123 +enum KimiK3Mode {
124 + /// Before any channel opens: channel opens are expected, but raw text
125 + /// falls through as visible text so marker-free output still streams.
126 + #[default]
127 + Idle,
128 + /// Inside the `think` channel.
129 + Reasoning,
130 + /// Inside the `response` channel.
131 + Response,
132 + /// After the `response` (or `tools`) channel closed: only a `tools`
133 + /// channel or the message close may follow, and noise is never content.
134 + Epilogue,
135 + /// Inside the `tools` channel, between `call` blocks.
136 + Tools,
137 + /// Inside one `call` block, buffering its body until the close marker.
138 + Call {
139 + name: String,
140 + index: Option<String>,
141 + scan: MarkerScanState,
142 + },
143 + /// After the message closed: ignore the rest (EOS leakage guard).
144 + Done,
145 +}
146 +
147 +/// Unified parser for Kimi K3 XTML think / response / tools channels.
148 +pub struct KimiK3UnifiedParser {
149 + buffer: String,
150 + mode: KimiK3Mode,
151 + /// Parser-provided tool-call IDs (`{tool}:{zero_based_index}`) by tool
152 + /// index; its length is also the count of emitted calls.
153 + call_ids: Vec<String>,
154 + tokenizer: DynTokenizer,
155 + open_token_id: u32,
156 + sep_token_id: u32,
157 +}
158 +
159 +impl KimiK3UnifiedParser {
160 + /// Create a Kimi K3 parser.
161 + pub fn new(_tools: &[Tool], tokenizer: DynTokenizer) -> Result<Self> {
162 + let open_token_id = token_id(tokenizer.as_ref(), OPEN)?;
163 + let sep_token_id = token_id(tokenizer.as_ref(), SEP)?;
164 +
165 + Ok(Self {
166 + buffer: String::new(),
167 + mode: KimiK3Mode::default(),
168 + call_ids: Vec::new(),
169 + tokenizer,
170 + open_token_id,
171 + sep_token_id,
172 + })
173 + }
174 +
175 + /// Detect the prefilled generation channel from the prompt tail.
176 + ///
177 + /// `add_generation_prompt` ends the prompt with `<|open|>think<|sep|>` or
178 + /// `<|open|>response<|sep|>`, so generation starts inside that channel and
179 + /// never re-emits the open tag. Locate the last `<|open|>…<|sep|>` pair and
180 + /// decode the short tag between the two structural tokens.
181 + fn initialize_mode(&mut self, prompt_token_ids: &[u32]) {
182 + self.mode = KimiK3Mode::Idle;
183 +
184 + let Some(sep_pos) = prompt_token_ids.iter().rposition(|&id| id == self.sep_token_id) else {
185 + return;
186 + };
187 + let Some(open_pos) =
188 + prompt_token_ids[..sep_pos].iter().rposition(|&id| id == self.open_token_id)
189 + else {
190 + return;
191 + };
192 + let tag_ids = &prompt_token_ids[open_pos + 1..sep_pos];
193 + if tag_ids.is_empty() || tag_ids.len() > MAX_PREFILL_TAG_TOKENS {
194 + return;
195 + }
196 + let Ok(tag) = self.tokenizer.decode(tag_ids, /* skip_special_tokens */ false) else {
197 + return;
198 + };
199 + self.mode = match tag.trim() {
200 + "think" => KimiK3Mode::Reasoning,
201 + "response" => KimiK3Mode::Response,
202 + _ => KimiK3Mode::Idle,
203 + };
204 + }
205 +
206 + fn apply_event(&mut self, event: KimiK3Event, output: &mut UnifiedParserOutput) -> Result<()> {
207 + match event {
208 + KimiK3Event::Text { len } => output.push_text(self.buffer[..len].to_string()),
209 + KimiK3Event::Reasoning { len } => {
210 + output.push_reasoning(self.buffer[..len].to_string());
211 + }
212 + KimiK3Event::Skip => {}
213 + KimiK3Event::ThinkOpen => self.mode = KimiK3Mode::Reasoning,
214 + KimiK3Event::ThinkClose => self.mode = KimiK3Mode::Idle,
215 + KimiK3Event::ResponseOpen => self.mode = KimiK3Mode::Response,
216 + KimiK3Event::ResponseClose => self.mode = KimiK3Mode::Epilogue,
217 + KimiK3Event::ToolsOpen => self.mode = KimiK3Mode::Tools,
218 + KimiK3Event::ToolsClose => self.mode = KimiK3Mode::Epilogue,
219 + KimiK3Event::MessageEnd => self.mode = KimiK3Mode::Done,
220 + KimiK3Event::CallOpen { name, index } => {
221 + self.mode = KimiK3Mode::Call {
222 + name,
223 + index,
224 + scan: MarkerScanState::default(),
225 + };
226 + }
227 + KimiK3Event::CallComplete { arguments } => {
228 + let mode = std::mem::replace(&mut self.mode, KimiK3Mode::Tools);
229 + let KimiK3Mode::Call { name, index, .. } = mode else {
230 + return Err(parsing_failed!(
231 + "Kimi K3 call completion without an active tool call"
232 + ));
233 + };
234 + // An empty/garbage call block without a tool name is dropped,
235 + // matching the Python parser.
236 + if name.is_empty() {
237 + return Ok(());
238 + }
239 +
240 + let tool_index = self.call_ids.len();
241 + self.call_ids.push(tool_call_id_for(&name, index.as_deref()));
242 + output.push_call(ToolCallDelta {
243 + tool_index,
244 + name: Some(name),
245 + arguments,
246 + });
247 + }
248 + }
249 + Ok(())
250 + }
251 +
252 + fn reset_state(&mut self) -> String {
253 + self.mode = KimiK3Mode::Idle;
254 + self.call_ids.clear();
255 + std::mem::take(&mut self.buffer)
256 + }
257 +}
258 +
259 +impl UnifiedParser for KimiK3UnifiedParser {
260 + fn create(tools: &[Tool], tokenizer: DynTokenizer) -> Result<Box<dyn UnifiedParser>>
261 + where
262 + Self: Sized + 'static,
263 + {
264 + Self::new(tools, tokenizer).map(|parser| Box::new(parser) as Box<dyn UnifiedParser>)
265 + }
266 +
267 + fn initialize(&mut self, prompt_token_ids: &[u32]) -> Result<()> {
268 + self.buffer.clear();
269 + self.call_ids.clear();
270 + self.initialize_mode(prompt_token_ids);
271 + Ok(())
272 + }
273 +
274 + fn preserve_special_tokens(&self) -> bool {
275 + true
276 + }
277 +
278 + fn structural_tag_builder(&self) -> Option<&dyn StructuralTagBuilder> {
279 + Some(&KIMI_K3_STRUCTURAL_TAG_BUILDER)
280 + }
281 +
282 + fn tool_call_id(&self, tool_index: usize) -> Option<&str> {
283 + self.call_ids.get(tool_index).map(String::as_str)
284 + }
285 +
286 + fn parse_into(&mut self, chunk: &str, output: &mut UnifiedParserOutput) -> Result<()> {
287 + self.buffer.push_str(chunk);
288 +
289 + while let Some((event, consumed_len)) = parse_buffered_event(&self.buffer, |input| {
290 + parse_next_kimi_k3_event(input, &mut self.mode)
291 + })? {
292 + self.apply_event(event, output)?;
293 + self.buffer.drain(..consumed_len);
294 + }
295 +
296 + Ok(())
297 + }
298 +
299 + fn finish(&mut self) -> Result<UnifiedParserOutput> {
300 + let mut output = UnifiedParserOutput::default();
301 +
302 + match &self.mode {
303 + KimiK3Mode::Idle | KimiK3Mode::Response => {
304 + output.push_text(std::mem::take(&mut self.buffer));
305 + }
306 + KimiK3Mode::Reasoning => output.push_reasoning(std::mem::take(&mut self.buffer)),
307 + KimiK3Mode::Epilogue | KimiK3Mode::Done => self.buffer.clear(),
308 + // A tools channel truncated between complete calls loses only its
309 + // closing markers; keep the calls already emitted.
310 + KimiK3Mode::Tools if self.buffer.is_empty() => {}
311 + KimiK3Mode::Tools | KimiK3Mode::Call { .. } => {
312 + return Err(parsing_failed!("incomplete Kimi K3 tool call"));
313 + }
314 + }
315 +
316 + // Keep call_ids so tool_call_id() stays available after the stream ends.
317 + self.mode = KimiK3Mode::Idle;
318 + Ok(output)
319 + }
320 +
321 + fn reset(&mut self) -> String {
322 + self.reset_state()
323 + }
324 +}
325 +
326 +/// Build the API-side tool-call ID from the XTML one-based `index` attribute.
327 +///
328 +/// The ID uses the zero-based call ordinal; XTML's message index stays
329 +/// one-based when rendering tool result messages.
330 +fn tool_call_id_for(name: &str, index: Option<&str>) -> String {
331 + match index {
332 + None => name.to_string(),
333 + Some(raw) => match raw.parse::<i64>() {
334 + Ok(one_based) => format!("{name}:{}", one_based - 1),
335 + Err(_) => format!("{name}:{raw}"),
336 + },
337 + }
338 +}
339 +
340 +/// Parse one Kimi K3 event from buffered streaming input.
341 +fn parse_next_kimi_k3_event(
342 + input: &mut KimiK3Input<'_>,
343 + mode: &mut KimiK3Mode,
344 +) -> ModalResult<KimiK3Event> {
345 + match mode {
346 + KimiK3Mode::Idle => parse_idle_event(input),
347 + KimiK3Mode::Reasoning => parse_reasoning_event(input),
348 + KimiK3Mode::Response => parse_response_event(input),
349 + KimiK3Mode::Epilogue => parse_epilogue_event(input),
350 + KimiK3Mode::Tools => parse_tools_event(input),
351 + KimiK3Mode::Call { scan, .. } => call_body_event(input, scan),
352 + KimiK3Mode::Done => parse_done_event(input),
353 + }
354 +}
355 +
356 +/// Parse an event while waiting for the next channel open.
357 +fn parse_idle_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
358 + alt((
359 + literal(THINK_OPEN).value(KimiK3Event::ThinkOpen),
360 + literal(RESPONSE_OPEN).value(KimiK3Event::ResponseOpen),
361 + literal(TOOLS_OPEN).value(KimiK3Event::ToolsOpen),
362 + message_end_event,
363 + safe_idle_text_event,
364 + ))
365 + .parse_next(input)
366 +}
367 +
368 +/// Parse an event inside the `think` channel.
369 +fn parse_reasoning_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
370 + alt((
371 + literal(THINK_CLOSE).value(KimiK3Event::ThinkClose),
372 + // `<|end_of_msg|>` can reach the parser under `ignore_eos` or
373 + // `include_stop_str_in_output`; never leak it into reasoning.
374 + literal(END_OF_MSG).value(KimiK3Event::MessageEnd),
375 + safe_reasoning_event,
376 + ))
377 + .parse_next(input)
378 +}
379 +
380 +/// Parse an event inside the `response` channel.
381 +fn parse_response_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
382 + alt((
383 + literal(RESPONSE_CLOSE).value(KimiK3Event::ResponseClose),
384 + // The response body also implicitly ends at a `tools` channel.
385 + literal(TOOLS_OPEN).value(KimiK3Event::ToolsOpen),
386 + message_end_event,
387 + safe_response_text_event,
388 + ))
389 + .parse_next(input)
390 +}
391 +
392 +/// Parse an event after the response channel closed.
393 +fn parse_epilogue_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
394 + alt((
395 + literal(TOOLS_OPEN).value(KimiK3Event::ToolsOpen),
396 + message_end_event,
397 + skip_epilogue_noise_event,
398 + ))
399 + .parse_next(input)
400 +}
401 +
402 +/// Parse an event inside the `tools` channel, between `call` blocks.
403 +fn parse_tools_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
404 + alt((
405 + call_open_event,
406 + literal(TOOLS_CLOSE).value(KimiK3Event::ToolsClose),
407 + // Defensive: an unterminated tools channel still ends with the message.
408 + message_end_event,
409 + skip_tools_noise_event,
410 + ))
411 + .parse_next(input)
412 +}
413 +
414 +/// Parse a message close or end-of-message marker.
415 +fn message_end_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
416 + alt((literal(MESSAGE_CLOSE), literal(END_OF_MSG)))
417 + .value(KimiK3Event::MessageEnd)
418 + .parse_next(input)
419 +}
420 +
421 +/// Ignore everything after the assistant message closed.
422 +fn parse_done_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
423 + rest.value(KimiK3Event::Skip).parse_next(input)
424 +}
425 +
426 +/// Parse safe text while waiting for the next channel marker.
427 +fn safe_idle_text_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
428 + safe_text_len_mul(input, IDLE_MARKERS).map(|len| KimiK3Event::Text { len })
429 +}
430 +
431 +/// Parse safe reasoning before the think close marker.
432 +fn safe_reasoning_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
433 + safe_text_len_mul(input, REASONING_MARKERS).map(|len| KimiK3Event::Reasoning { len })
434 +}
435 +
436 +/// Parse safe response text before the next channel marker.
437 +fn safe_response_text_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
438 + safe_text_len_mul(input, RESPONSE_MARKERS).map(|len| KimiK3Event::Text { len })
439 +}
440 +
441 +/// Skip non-content noise after the response channel closed.
442 +fn skip_epilogue_noise_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
443 + safe_text_len_mul(input, EPILOGUE_MARKERS).map(|_| KimiK3Event::Skip)
444 +}
445 +
446 +/// Skip non-content noise between `call` blocks.
447 +fn skip_tools_noise_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
448 + safe_text_len_mul(input, TOOLS_MARKERS).map(|_| KimiK3Event::Skip)
449 +}
450 +
451 +/// Parse a `call` open tag into its tool name and one-based index.
452 +fn call_open_event(input: &mut KimiK3Input<'_>) -> ModalResult<KimiK3Event> {
453 + let (attrs,) = seq!(
454 + _: literal(CALL_OPEN),
455 + take_until(0.., SEP),
456 + _: literal(SEP),
457 + )
458 + .parse_next(input)?;
459 + let attrs = parse_tag_attrs(attrs)?;
460 +
461 + Ok(KimiK3Event::CallOpen {
462 + name: attr_value(&attrs, "tool").unwrap_or_default().to_string(),
463 + index: attr_value(&attrs, "index")
464 + .filter(|index| !index.is_empty())
465 + .map(str::to_string),
466 + })
467 +}
468 +
469 +/// Parse the buffered call body through `<|close|>call<|sep|>` into a
470 +/// completed call.
471 +fn call_body_event(
472 + input: &mut KimiK3Input<'_>,
473 + scan: &mut MarkerScanState,
474 +) -> ModalResult<KimiK3Event> {
475 + let (body,) = seq!(
476 + take_until_marker(CALL_CLOSE, scan),
477 + _: literal(CALL_CLOSE),
478 + )
479 + .parse_next(input)?;
480 + let arguments = parse_call_arguments(body)?;
481 +
482 + Ok(KimiK3Event::CallComplete { arguments })
483 +}
484 +
485 +/// Parse a complete `call` body into the OpenAI-style arguments JSON string.
486 +///
487 +/// The body is either one raw `json` block (passed through unmodified) or a
488 +/// sequence of typed `argument` blocks (converted per their `type` tags).
489 +fn parse_call_arguments(body: &str) -> ModalResult<String> {
490 + let mut input = body;
491 + terminated(
492 + delimited(ws0, alt((json_block_arguments, typed_arguments)), ws0),
493 + eof,
494 + )
495 + .parse_next(&mut input)
496 + .map_err(|_| xtml_error("Kimi K3 call body"))
497 +}
498 +
499 +/// Parse one raw `json` argument block, passing its body through unmodified.
500 +fn json_block_arguments(input: &mut &str) -> ModalResult<String> {
501 + let (raw,) = seq!(
502 + _: literal(JSON_OPEN),
503 + _: take_until(0.., SEP), // attrs (`type="object"`), unused on decode
504 + _: literal(SEP),
505 + take_until(0.., JSON_CLOSE),
506 + _: literal(JSON_CLOSE),
507 + )
508 + .parse_next(input)?;
509 + Ok(raw.to_string())
510 +}
511 +
512 +/// Parse typed `argument` blocks into a serialized JSON object, preserving
513 +/// argument order.
514 +fn typed_arguments(input: &mut &str) -> ModalResult<String> {
515 + let pairs: Vec<(String, Value)> =
516 + repeat(0.., terminated(argument_block, ws0)).parse_next(input)?;
517 + let arguments = pairs.into_iter().collect::<Map<String, Value>>();
518 + serde_json::to_string(&arguments).map_err(|_| xtml_error("Kimi K3 arguments"))
519 +}
520 +
521 +/// Parse one typed `argument` block into its key/value pair.
522 +fn argument_block(input: &mut &str) -> ModalResult<(String, Value)> {
523 + let (attrs, raw) = seq!(
524 + _: literal(ARG_OPEN),
525 + take_until(0.., SEP),
526 + _: literal(SEP),
527 + take_until(0.., ARG_CLOSE),
528 + _: literal(ARG_CLOSE),
529 + )
530 + .parse_next(input)?;
531 + let attrs = parse_tag_attrs(attrs)?;
532 +
533 + let key = attr_value(&attrs, "key").unwrap_or_default().to_string();
534 + let arg_type = attr_value(&attrs, "type").unwrap_or("string");
535 + Ok((key, decode_argument_value(arg_type, raw)))
536 +}
537 +
538 +/// Decode one typed argument value per its XTML `type` tag.
539 +///
540 +/// `string` values pass the raw text through (the renderer emits them
541 +/// unescaped); other types are JSON-decoded, falling back to the raw text on
542 +/// malformed payloads so one quirky value does not fail the whole call.
543 +fn decode_argument_value(arg_type: &str, raw: &str) -> Value {
544 + if arg_type == "string" {
545 + return Value::String(raw.to_string());
546 + }
547 + serde_json::from_str(raw).unwrap_or_else(|_| Value::String(raw.to_string()))
548 +}
549 +
550 +/// Parse a complete XTML attribute string like ` tool="get_weather" index="1"`.
551 +fn parse_tag_attrs(attrs: &str) -> ModalResult<Vec<(String, String)>> {
552 + let mut input = attrs;
553 + terminated(repeat(0.., preceded(ws1, tag_attr)), (ws0, eof))
554 + .parse_next(&mut input)
555 + .map_err(|_| xtml_error("XTML tag attributes"))
556 +}
557 +
558 +/// Parse one XTML `key="value"` attribute pair.
559 +fn tag_attr(input: &mut &str) -> ModalResult<(String, String)> {
560 + seq!(
561 + take_while(1.., |char: char| char.is_alphanumeric() || char == '_').map(str::to_string),
562 + _: literal("=\""),
563 + take_till(0.., '"').map(unescape_attr_value),
564 + _: literal("\""),
565 + )
566 + .parse_next(input)
567 +}
568 +
569 +/// Reverse XTML attribute escaping: `&quot;` first, then `&amp;` (the inverse
570 +/// of the encode order).
571 +fn unescape_attr_value(value: &str) -> String {
572 + value.replace("&quot;", "\"").replace("&amp;", "&")
573 +}
574 +
575 +/// Look up one parsed attribute value by key.
576 +fn attr_value<'a>(attrs: &'a [(String, String)], key: &str) -> Option<&'a str> {
577 + attrs.iter().find(|(name, _)| name == key).map(|(_, value)| value.as_str())
578 +}
579 +
580 +/// Build a cut error for determinably malformed XTML structure.
581 +fn xtml_error(label: &'static str) -> ErrMode<ContextError> {
582 + let mut error = ContextError::new();
583 + error.push(StrContext::Label(label));
584 + ErrMode::Cut(error)
585 +}
586 +
587 +#[cfg(test)]
588 +mod tests {
589 + use std::sync::Arc;
590 +
591 + use serde_json::{Value, json};
592 + use thiserror_ext::AsReport;
593 + use vllm_tokenizer::Tokenizer as _;
594 + use vllm_tokenizer::test_utils::TestTokenizer;
595 +
596 + use super::{
597 + END_OF_MSG, KimiK3UnifiedParser, OPEN, RESPONSE_CLOSE, RESPONSE_OPEN, SEP, THINK_CLOSE,
598 + THINK_OPEN, TOOLS_CLOSE, TOOLS_OPEN,
599 + };
600 + use crate::tool::ToolCallDelta;
601 + use crate::unified::{
602 + UnifiedParser, UnifiedParserError, UnifiedParserEvent, UnifiedParserOutput,
603 + };
604 +
605 + const OPEN_ID: u32 = 256;
606 + const CLOSE_ID: u32 = 257;
607 + const SEP_ID: u32 = 258;
608 + const END_OF_MSG_ID: u32 = 259;
609 +
610 + fn tokenizer() -> TestTokenizer {
611 + TestTokenizer::new()
612 + .with_special_token(OPEN, OPEN_ID)
613 + .with_special_token("<|close|>", CLOSE_ID)
614 + .with_special_token(SEP, SEP_ID)
615 + .with_special_token(END_OF_MSG, END_OF_MSG_ID)
616 + }
617 +
618 + trait UnifiedParserTestExt {
619 + fn parse_chunk(&mut self, chunk: &str) -> super::Result<UnifiedParserOutput>;
620 + fn parse_complete(&mut self, text: &str) -> super::Result<UnifiedParserOutput>;
621 + }
622 +
623 + impl UnifiedParserTestExt for KimiK3UnifiedParser {
624 + fn parse_chunk(&mut self, chunk: &str) -> super::Result<UnifiedParserOutput> {
625 + let mut output = UnifiedParserOutput::default();
626 + self.parse_into(chunk, &mut output)?;
627 + Ok(output)
628 + }
629 +
630 + fn parse_complete(&mut self, text: &str) -> super::Result<UnifiedParserOutput> {
631 + let mut output = self.parse_chunk(text)?;
632 + output.append(self.finish()?);
633 + Ok(output)
634 + }
635 + }
636 +
637 + trait UnifiedOutputTestExt {
638 + fn normal_text(&self) -> String;
639 + fn reasoning_text(&self) -> String;
640 + fn calls(&self) -> Vec<ToolCallDelta>;
641 + }
642 +
643 + impl UnifiedOutputTestExt for UnifiedParserOutput {
644 + fn normal_text(&self) -> String {
645 + self.events
646 + .iter()
647 + .filter_map(|event| match event {
648 + UnifiedParserEvent::Text(text) => Some(text.as_str()),
649 + _ => None,
650 + })
651 + .collect()
652 + }
653 +
654 + fn reasoning_text(&self) -> String {
655 + self.events
656 + .iter()
657 + .filter_map(|event| match event {
658 + UnifiedParserEvent::Reasoning(text) => Some(text.as_str()),
659 + _ => None,
660 + })
661 + .collect()
662 + }
663 +
664 + fn calls(&self) -> Vec<ToolCallDelta> {
665 + self.events
666 + .iter()
667 + .filter_map(|event| match event {
668 + UnifiedParserEvent::ToolCall(call) => Some(call.clone()),
669 + _ => None,
670 + })
671 + .collect()
672 + }
673 + }
674 +
675 + fn test_parser() -> KimiK3UnifiedParser {
676 + KimiK3UnifiedParser::new(&[], Arc::new(tokenizer())).unwrap()
677 + }
678 +
679 + fn collect_stream(parser: &mut KimiK3UnifiedParser, chunks: &[&str]) -> UnifiedParserOutput {
680 + let mut output = UnifiedParserOutput::default();
681 + for chunk in chunks {
682 + output.append(parser.parse_chunk(chunk).unwrap());
683 + }
684 + output.append(parser.finish().unwrap());
685 + output
686 + }
687 +
688 + /// Split `text` into small chunks to stress marker-split handling.
689 + fn char_chunks(text: &str, size: usize) -> Vec<String> {
690 + let chars: Vec<char> = text.chars().collect();
691 + chars.chunks(size).map(|chunk| chunk.iter().collect()).collect()
692 + }
693 +
694 + fn arg(key: &str, arg_type: &str, value: &str) -> String {
695 + format!(
696 + "{OPEN}argument key=\"{key}\" type=\"{arg_type}\"{SEP}{value}<|close|>argument{SEP}"
697 + )
698 + }
699 +
700 + fn call(attrs: &str, body: &str) -> String {
701 + format!("{OPEN}call {attrs}{SEP}{body}<|close|>call{SEP}")
702 + }
703 +
704 + fn thinking_output(reasoning: &str, response: &str, tools_body: &str) -> String {
705 + let mut output = format!("{THINK_OPEN}{reasoning}{THINK_CLOSE}");
706 + output.push_str(&format!("{RESPONSE_OPEN}{response}{RESPONSE_CLOSE}"));
707 + if !tools_body.is_empty() {
708 + output.push_str(&format!("{TOOLS_OPEN}{tools_body}{TOOLS_CLOSE}"));
709 + }
710 + output.push_str("<|close|>message<|sep|>");
711 + output
712 + }
713 +
714 + fn first_call(output: &UnifiedParserOutput) -> ToolCallDelta {
715 + output.calls().first().expect("expected one tool call").clone()
716 + }
717 +
718 + #[test]
719 + fn kimi_k3_create_requires_structural_tokens() {
720 + let error = match KimiK3UnifiedParser::new(&[], Arc::new(TestTokenizer::new())) {
721 + Ok(_) => panic!("expected missing token error"),
722 + Err(error) => error,
723 + };
724 +
725 + assert!(matches!(
726 + error,
727 + UnifiedParserError::MissingToken { token } if token == OPEN
728 + ));
729 + }
730 +
731 + #[test]
732 + fn kimi_k3_parses_reasoning_response_and_typed_tool_call() {
733 + let body = [
734 + arg("city", "string", "Hangzhou"),
735 + arg("days", "number", "1.5"),
736 + arg("detailed", "boolean", "true"),
737 + arg("filters", "object", r#"{"kind":"rain"}"#),
738 + arg("hours", "array", "[8,20]"),
739 + ]
740 + .concat();
741 + let text = thinking_output(
742 + "Need the weather tool.",
743 + "I'll check.",
744 + &call("tool=\"get_weather\" index=\"1\"", &body),
745 + );
746 +
747 + let mut parser = test_parser();
748 + let output = parser.parse_complete(&text).unwrap();
749 +
750 + assert_eq!(output.reasoning_text(), "Need the weather tool.");
751 + assert_eq!(output.normal_text(), "I'll check.");
752 + let call = first_call(&output);
753 + assert_eq!(call.tool_index, 0);
754 + assert_eq!(call.name.as_deref(), Some("get_weather"));
755 + assert_eq!(
756 + serde_json::from_str::<Value>(&call.arguments).unwrap(),
757 + json!({
758 + "city": "Hangzhou",
759 + "days": 1.5,
760 + "detailed": true,
761 + "filters": { "kind": "rain" },
762 + "hours": [8, 20],
763 + })
764 + );
765 + assert_eq!(parser.tool_call_id(0), Some("get_weather:0"));
766 + }
767 +
768 + #[test]
769 + fn kimi_k3_arguments_preserve_order_and_number_formatting() {
770 + let body = [
771 + arg("y", "number", "1.0"),
772 + arg("x", "number", "2"),
773 + arg("items", "array", r#"["left","right"]"#),
774 + ]
775 + .concat();
776 + let text = thinking_output("t", "", &call("tool=\"add\" index=\"1\"", &body));
777 +
778 + let mut parser = test_parser();
779 + let output = parser.parse_complete(&text).unwrap();
780 +
781 + assert_eq!(
782 + first_call(&output).arguments,
783 + r#"{"y":1.0,"x":2,"items":["left","right"]}"#
784 + );
785 + }
786 +
787 + #[test]
788 + fn kimi_k3_streaming_splits_markers_across_chunks() {
789 + let text = thinking_output(
790 + "step by step",
791 + "the answer",
792 + &call("tool=\"calc\" index=\"1\"", &arg("x", "number", "42")),
793 + );
794 +
795 + for size in [1, 3, 7] {
796 + let chunks = char_chunks(&text, size);
797 + let chunk_refs: Vec<&str> = chunks.iter().map(String::as_str).collect();
798 + let output = collect_stream(&mut test_parser(), &chunk_refs);
799 +
800 + assert_eq!(output.reasoning_text(), "step by step", "chunk size {size}");
801 + assert_eq!(output.normal_text(), "the answer", "chunk size {size}");
802 + assert_eq!(first_call(&output).name.as_deref(), Some("calc"));
803 + assert_eq!(first_call(&output).arguments, r#"{"x":42}"#);
804 + }
805 + }
806 +
807 + #[test]
808 + fn kimi_k3_streaming_emits_text_incrementally() {
809 + let mut parser = test_parser();
810 + let prompt = tokenizer().encode("<|open|>response<|sep|>", false).unwrap();
811 + parser.initialize(&prompt).unwrap();
812 +
813 + let first = parser.parse_chunk("Hel").unwrap();
814 + assert_eq!(first.normal_text(), "Hel");
815 +
816 + let second = parser.parse_chunk("lo<|close|>resp").unwrap();
817 + assert_eq!(second.normal_text(), "lo");
818 +
819 + let mut output = parser.parse_chunk("onse<|sep|>").unwrap();
820 + output.append(parser.finish().unwrap());
821 + assert_eq!(output.normal_text(), "");
822 + }
823 +
824 + #[test]
825 + fn kimi_k3_initialize_think_prefill_starts_in_reasoning() {
826 + let mut parser = test_parser();
827 + let prompt = tokenizer()
828 + .encode(
829 + "<|open|>message role=\"assistant\"<|sep|><|open|>think<|sep|>",
830 + false,
831 + )
832 + .unwrap();
833 + parser.initialize(&prompt).unwrap();
834 +
835 + let output = parser
836 + .parse_complete(&format!(
837 + "reasoning{THINK_CLOSE}{RESPONSE_OPEN}answer{RESPONSE_CLOSE}<|close|>message{SEP}"
838 + ))
839 + .unwrap();
840 +
841 + assert_eq!(output.reasoning_text(), "reasoning");
842 + assert_eq!(output.normal_text(), "answer");
843 + }
844 +
845 + #[test]
846 + fn kimi_k3_initialize_response_prefill_starts_in_response() {
847 + let mut parser = test_parser();
848 + let prompt = tokenizer()
849 + .encode(
850 + "<|open|>message role=\"assistant\"<|sep|><|open|>response<|sep|>",
851 + false,
852 + )
853 + .unwrap();
854 + parser.initialize(&prompt).unwrap();
855 +
856 + let output = parser
857 + .parse_complete(&format!("answer{RESPONSE_CLOSE}<|close|>message{SEP}"))
858 + .unwrap();
859 +
860 + assert_eq!(output.normal_text(), "answer");
861 + assert!(output.reasoning_text().is_empty());
862 + }
863 +
864 + #[test]
865 + fn kimi_k3_initialize_message_open_prefill_starts_idle() {
866 + let mut parser = test_parser();
867 + let prompt =
868 + tokenizer().encode("<|open|>message role=\"assistant\"<|sep|>", false).unwrap();
869 + parser.initialize(&prompt).unwrap();
870 +
871 + let output = parser
872 + .parse_complete(&format!("{THINK_OPEN}reason{THINK_CLOSE}{RESPONSE_OPEN}hi"))
873 + .unwrap();
874 +
875 + assert_eq!(output.reasoning_text(), "reason");
876 + assert_eq!(output.normal_text(), "hi");
877 + }
878 +
879 + #[test]
880 + fn kimi_k3_plain_text_falls_through_as_text() {
881 + let output = collect_stream(&mut test_parser(), &["plain ", "answer"]);
882 +
883 + assert_eq!(output.normal_text(), "plain answer");
884 + assert!(output.reasoning_text().is_empty());
885 + assert!(output.calls().is_empty());
886 + }
887 +
888 + #[test]
889 + fn kimi_k3_tool_call_waits_for_close_marker() {
890 + let mut parser = test_parser();
891 + let mut output = UnifiedParserOutput::default();
892 +
893 + let argument = arg("x", "number", "1");
894 + for chunk in [
895 + TOOLS_OPEN,
896 + "<|open|>call tool=\"calc\" index=\"1\"<|sep|>",
897 + argument.as_str(),
898 + ] {
899 + output.append(parser.parse_chunk(chunk).unwrap());
900 + assert!(output.calls().is_empty());
901 + }
902 +
903 + output.append(parser.parse_chunk("<|close|>call<|sep|>").unwrap());
904 +
905 + assert_eq!(first_call(&output).name.as_deref(), Some("calc"));
906 + assert_eq!(first_call(&output).arguments, r#"{"x":1}"#);
907 + }
908 +
909 + #[test]
910 + fn kimi_k3_parses_multiple_tool_calls() {
911 + let tools_body = format!(
912 + "{}{}",
913 + call(
914 + "tool=\"get_weather\" index=\"1\"",
915 + &arg("city", "string", "SF")
916 + ),
917 + call("tool=\"get_time\" index=\"2\"", ""),
918 + );
919 + let text = thinking_output("t", "r", &tools_body);
920 +
921 + let mut parser = test_parser();
922 + let output = parser.parse_complete(&text).unwrap();
923 +
924 + let calls = output.calls();
925 + assert_eq!(calls.len(), 2);
926 + assert_eq!(calls[0].tool_index, 0);
927 + assert_eq!(calls[0].name.as_deref(), Some("get_weather"));
928 + assert_eq!(calls[0].arguments, r#"{"city":"SF"}"#);
929 + assert_eq!(calls[1].tool_index, 1);
930 + assert_eq!(calls[1].name.as_deref(), Some("get_time"));
931 + assert_eq!(calls[1].arguments, "{}");
932 + assert_eq!(parser.tool_call_id(0), Some("get_weather:0"));
933 + assert_eq!(parser.tool_call_id(1), Some("get_time:1"));
934 + }
935 +
936 + #[test]
937 + fn kimi_k3_json_block_arguments_pass_through_raw() {
938 + // Spacing and key order must survive unmodified: raw `json` blocks are
939 + // not validated or normalized.
940 + let raw = r#"{"b": 1, "a": [2 , 3]}"#;
941 + let body = format!("{OPEN}json type=\"object\"{SEP}{raw}<|close|>json{SEP}");
942 + let text = thinking_output("t", "", &call("tool=\"run\" index=\"1\"", &body));
943 +
944 + let mut parser = test_parser();
945 + let output = parser.parse_complete(&text).unwrap();
946 +
947 + assert_eq!(first_call(&output).arguments, raw);
948 + }
949 +
950 + #[test]
951 + fn kimi_k3_string_argument_passes_raw_text_through() {
952 + let value = "line one\nline two {\"not\": \"json\"} & <tags>";
953 + let text = thinking_output(
954 + "t",
955 + "",
956 + &call(
957 + "tool=\"write\" index=\"1\"",
958 + &arg("content", "string", value),
959 + ),
960 + );
961 +
962 + let mut parser = test_parser();
963 + let output = parser.parse_complete(&text).unwrap();
964 +
965 + assert_eq!(
966 + serde_json::from_str::<Value>(&first_call(&output).arguments).unwrap(),
967 + json!({ "content": value })
968 + );
969 + }
970 +
971 + #[test]
972 + fn kimi_k3_malformed_typed_argument_falls_back_to_raw_text() {
973 + let text = thinking_output(
974 + "t",
975 + "",
976 + &call(
977 + "tool=\"calc\" index=\"1\"",
978 + &arg("x", "number", "not a number"),
979 + ),
980 + );
981 +
982 + let mut parser = test_parser();
983 + let output = parser.parse_complete(&text).unwrap();
984 +
985 + assert_eq!(first_call(&output).arguments, r#"{"x":"not a number"}"#);
986 + }
987 +
988 + #[test]
989 + fn kimi_k3_attribute_values_are_unescaped() {
990 + let text = thinking_output(
991 + "t",
992 + "",
993 + &call(
994 + "tool=\"a&quot;b&amp;c\" index=\"1\"",
995 + &arg("key", "string", "value"),
996 + ),
997 + );
998 +
999 + let mut parser = test_parser();
1000 + let output = parser.parse_complete(&text).unwrap();
1001 +
1002 + assert_eq!(first_call(&output).name.as_deref(), Some("a\"b&c"));
1003 + }
1004 +
1005 + #[test]
1006 + fn kimi_k3_call_without_tool_name_is_dropped() {
1007 + let tools_body = format!(
1008 + "{}{}",
1009 + call("index=\"1\"", &arg("x", "number", "1")),
1010 + call("tool=\"real\" index=\"2\"", ""),
1011 + );
1012 + let text = thinking_output("t", "", &tools_body);
1013 +
1014 + let mut parser = test_parser();
1015 + let output = parser.parse_complete(&text).unwrap();
1016 +
1017 + let calls = output.calls();
1018 + assert_eq!(calls.len(), 1);
1019 + assert_eq!(calls[0].tool_index, 0);
1020 + assert_eq!(calls[0].name.as_deref(), Some("real"));
1021 + assert_eq!(parser.tool_call_id(0), Some("real:1"));
1022 + }
1023 +
1024 + #[test]
1025 + fn kimi_k3_tool_call_id_follows_index_attribute() {
1026 + let tools_body = format!(
1027 + "{}{}{}",
1028 + call("tool=\"first\" index=\"3\"", ""),
1029 + call("tool=\"second\"", ""),
1030 + call("tool=\"third\" index=\"x\"", ""),
1031 + );
1032 + let text = thinking_output("t", "", &tools_body);
1033 +
1034 + let mut parser = test_parser();
1035 + parser.parse_complete(&text).unwrap();
1036 +
1037 + assert_eq!(parser.tool_call_id(0), Some("first:2"));
1038 + assert_eq!(parser.tool_call_id(1), Some("second"));
1039 + assert_eq!(parser.tool_call_id(2), Some("third:x"));
1040 + }
1041 +
1042 + #[test]
1043 + fn kimi_k3_ignores_output_after_message_close() {
1044 + let mut parser = test_parser();
1045 + let output = parser
1046 + .parse_complete(&format!(
1047 + "{RESPONSE_OPEN}answer{RESPONSE_CLOSE}<|close|>message{SEP}junk{END_OF_MSG}"
1048 + ))
1049 + .unwrap();
1050 +
1051 + assert_eq!(output.normal_text(), "answer");
1052 + }
1053 +
1054 + #[test]
1055 + fn kimi_k3_epilogue_noise_is_not_content() {
1056 + let text = format!(
1057 + "{RESPONSE_OPEN}answer{RESPONSE_CLOSE}\n{TOOLS_OPEN}{}{TOOLS_CLOSE}\n<|close|>message{SEP}",
1058 + call("tool=\"calc\" index=\"1\"", ""),
1059 + );
1060 +
1061 + let mut parser = test_parser();
1062 + let output = parser.parse_complete(&text).unwrap();
1063 +
1064 + assert_eq!(output.normal_text(), "answer");
1065 + assert_eq!(output.calls().len(), 1);
1066 + }
1067 +
1068 + #[test]
1069 + fn kimi_k3_finish_flushes_unclosed_reasoning() {
1070 + let mut parser = test_parser();
1071 + let mut output = parser.parse_chunk(&format!("{THINK_OPEN}still thinking")).unwrap();
1072 + output.append(parser.finish().unwrap());
1073 +
1074 + assert_eq!(output.reasoning_text(), "still thinking");
1075 + assert!(output.normal_text().is_empty());
1076 + }
1077 +
1078 + #[test]
1079 + fn kimi_k3_finish_flushes_partial_marker_as_text() {
1080 + let mut parser = test_parser();
1081 + let mut output = parser.parse_chunk("answer<|clo").unwrap();
1082 + output.append(parser.finish().unwrap());
1083 +
1084 + assert_eq!(output.normal_text(), "answer<|clo");
1085 + }
1086 +
1087 + #[test]
1088 + fn kimi_k3_finish_fails_mid_tool_call() {
1089 + let mut parser = test_parser();
1090 + parser
1091 + .parse_chunk(&format!(
1092 + "{TOOLS_OPEN}<|open|>call tool=\"calc\" index=\"1\"<|sep|>{}",
1093 + arg("x", "number", "1")
1094 + ))
1095 + .unwrap();
1096 +
1097 + let error = parser.finish().unwrap_err();
1098 +
1099 + assert!(error.to_report_string().contains("incomplete Kimi K3 tool call"));
1100 + }
1101 +
1102 + #[test]
1103 + fn kimi_k3_finish_after_truncated_tools_keeps_complete_calls() {
1104 + let mut parser = test_parser();
1105 + let mut output = parser
1106 + .parse_chunk(&format!(
1107 + "{TOOLS_OPEN}{}",
1108 + call("tool=\"calc\" index=\"1\"", &arg("x", "number", "1"))
1109 + ))
1110 + .unwrap();
1111 + output.append(parser.finish().unwrap());
1112 +
1113 + assert_eq!(first_call(&output).name.as_deref(), Some("calc"));
1114 + }
1115 +
1116 + #[test]
1117 + fn kimi_k3_malformed_call_attributes_fail_fast() {
1118 + let mut parser = test_parser();
1119 + let error = parser
1120 + .parse_chunk(&format!("{TOOLS_OPEN}<|open|>call garbage attrs<|sep|>"))
1121 + .unwrap_err();
1122 +
1123 + assert!(error.to_report_string().contains("XTML tag attributes"));
1124 + }
1125 +
1126 + #[test]
1127 + fn kimi_k3_empty_response_channel_emits_nothing() {
1128 + let text = thinking_output("t", "", &call("tool=\"calc\" index=\"1\"", ""));
1129 +
1130 + let mut parser = test_parser();
1131 + let output = parser.parse_complete(&text).unwrap();
1132 +
1133 + assert!(output.normal_text().is_empty());
1134 + assert_eq!(output.reasoning_text(), "t");
1135 + assert_eq!(output.calls().len(), 1);
1136 + }
1137 +
1138 + #[test]
1139 + fn kimi_k3_reset_returns_buffered_text() {
1140 + let mut parser = test_parser();
1141 + let prompt = tokenizer().encode("<|open|>response<|sep|>", false).unwrap();
1142 + parser.initialize(&prompt).unwrap();
1143 + parser.parse_chunk("answer<|close|>resp").unwrap();
1144 +
1145 + let raw = parser.reset();
1146 +
1147 + assert_eq!(raw, "<|close|>resp");
1148 + }
1149 +}
rust/src/parser/src/unified/kimi_k3/structural_tag.rs new
+503
@@ -0,0 +1,503 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +//! Structural-tag grammar for Kimi K3 XTML tool calls.
5 +
6 +use serde_json::{Map, Value};
7 +use xgrammar_structural_tag::Result;
8 +use xgrammar_structural_tag::builders::{
9 + StructuralTagBuilder, StructuralTagContext, StructuralTagOptions,
10 +};
11 +use xgrammar_structural_tag::format::{Format, JsonSchemaFormat, StructuralTag, TagFormat};
12 +use xgrammar_structural_tag::tool::{BuilderToolChoice, FunctionToolParam, function_parameters};
13 +
14 +use super::{
15 + ARG_CLOSE, CALL_CLOSE, END_OF_MSG, JSON_CLOSE, JSON_OPEN, MESSAGE_CLOSE, OPEN, RESPONSE_CLOSE,
16 + RESPONSE_OPEN, SEP, THINK_CLOSE, TOOLS_CLOSE, TOOLS_OPEN,
17 +};
18 +
19 +pub(super) static KIMI_K3_STRUCTURAL_TAG_BUILDER: KimiK3StructuralTagBuilder =
20 + KimiK3StructuralTagBuilder;
21 +
22 +const XTML_TYPES: &[&str] = &["string", "number", "boolean", "null", "object", "array"];
23 +
24 +/// Kimi K3 XTML structural-tag builder.
25 +#[derive(Debug, Clone, Copy, Default)]
26 +pub struct KimiK3StructuralTagBuilder;
27 +
28 +impl StructuralTagBuilder for KimiK3StructuralTagBuilder {
29 + fn build(&self, ctx: StructuralTagContext<'_>) -> Result<StructuralTag> {
30 + let mut elements = response_prefix(ctx.options.reasoning);
31 +
32 + let tools = match ctx.tool_choice {
33 + // Serving lowering filters empty tools before calling the builder,
34 + // while direct `build_structural_tag(..., auto, ...)` calls do not.
35 + BuilderToolChoice::Auto if ctx.function_tools.is_empty() => None,
36 + BuilderToolChoice::Auto => Some(Format::optional(tools_channel(
37 + ctx.function_tools,
38 + ctx.options,
39 + ))),
40 + BuilderToolChoice::Forced | BuilderToolChoice::Required => {
41 + Some(tools_channel(ctx.function_tools, ctx.options))
42 + }
43 + };
44 + if let Some(tools) = tools {
45 + elements.push(tools);
46 + }
47 + elements.push(Format::optional(Format::const_string(MESSAGE_CLOSE)));
48 +
49 + Ok(StructuralTag::new(Format::sequence(elements)))
50 + }
51 +}
52 +
53 +fn response_prefix(reasoning: bool) -> Vec<Format> {
54 + let mut elements = Vec::new();
55 + if reasoning {
56 + elements.push(Format::tag(
57 + "",
58 + Format::any_text_excluding(&[THINK_CLOSE, END_OF_MSG]),
59 + THINK_CLOSE,
60 + ));
61 + elements.push(Format::const_string(RESPONSE_OPEN));
62 + } else {
63 + elements.push(Format::optional(Format::const_string(RESPONSE_OPEN)));
64 + }
65 + elements.push(Format::tag(
66 + "",
67 + Format::any_text_excluding(&[RESPONSE_CLOSE, TOOLS_OPEN, MESSAGE_CLOSE, END_OF_MSG]),
68 + RESPONSE_CLOSE,
69 + ));
70 + elements
71 +}
72 +
73 +fn tools_channel(tools: &[FunctionToolParam], options: StructuralTagOptions) -> Format {
74 + let calls = tools.iter().map(|tool| call_tag(tool, options)).collect();
75 + Format::tag(
76 + TOOLS_OPEN,
77 + Format::tags_with_separator(calls, "", true, false),
78 + TOOLS_CLOSE,
79 + )
80 +}
81 +
82 +fn call_tag(tool: &FunctionToolParam, options: StructuralTagOptions) -> TagFormat {
83 + let parameters = function_parameters(&tool.function);
84 + let call_body = Format::or(vec![
85 + typed_arguments(&parameters, options),
86 + raw_json_arguments(&parameters, options),
87 + ]);
88 +
89 + TagFormat::new(
90 + format!(
91 + "{OPEN}call tool=\"{}\" index=\"",
92 + escape_attr_value(&tool.function.name)
93 + ),
94 + Format::sequence(vec![
95 + Format::regex("[1-9][0-9]*"),
96 + Format::const_string(format!("\"{SEP}")),
97 + call_body,
98 + ]),
99 + CALL_CLOSE,
100 + )
101 +}
102 +
103 +fn typed_arguments(parameters: &Value, options: StructuralTagOptions) -> Format {
104 + let Some(schema) = parameters.as_object() else {
105 + return if parameters == &Value::Bool(false) {
106 + Format::const_string("")
107 + } else {
108 + Format::star(permissive_argument())
109 + };
110 + };
111 + let Some(properties) = schema.get("properties").and_then(Value::as_object) else {
112 + return Format::star(permissive_argument());
113 + };
114 + if properties.is_empty() {
115 + return Format::star(permissive_argument());
116 + }
117 +
118 + let root_defs = root_definitions(schema);
119 + let arguments = properties
120 + .iter()
121 + .flat_map(|(key, schema)| argument_tags(key, schema, &root_defs, options))
122 + .map(Format::Tag)
123 + .collect::<Vec<_>>();
124 + let arguments = match arguments.as_slice() {
125 + [argument] => argument.clone(),
126 + _ => Format::or(arguments),
127 + };
128 + // Keep typed arguments order-agnostic and non-unique, but do not allow an
129 + // empty call when the root schema declares required properties.
130 + if schema
131 + .get("required")
132 + .and_then(Value::as_array)
133 + .is_some_and(|required| !required.is_empty())
134 + {
135 + Format::plus(arguments)
136 + } else {
137 + Format::star(arguments)
138 + }
139 +}
140 +
141 +fn argument_tags(
142 + key: &str,
143 + schema: &Value,
144 + root_defs: &Map<String, Value>,
145 + options: StructuralTagOptions,
146 +) -> Vec<TagFormat> {
147 + let types = schema_types(schema);
148 + types
149 + .into_iter()
150 + .map(|xtml_type| {
151 + let content = if xtml_type == "string" {
152 + string_argument_content(schema)
153 + } else {
154 + json_schema(
155 + attach_root_definitions(&narrow_schema_type(schema, xtml_type), root_defs),
156 + options,
157 + )
158 + };
159 + TagFormat::new(
160 + format!(
161 + "{OPEN}argument key=\"{}\" type=\"{xtml_type}\"{SEP}",
162 + escape_attr_value(key)
163 + ),
164 + content,
165 + ARG_CLOSE,
166 + )
167 + })
168 + .collect()
169 +}
170 +
171 +fn schema_types(schema: &Value) -> Vec<&'static str> {
172 + let Some(schema) = schema.as_object() else {
173 + return XTML_TYPES.to_vec();
174 + };
175 + let mut types = Vec::new();
176 + match schema.get("type") {
177 + Some(Value::String(value)) => push_schema_type(&mut types, value),
178 + Some(Value::Array(values)) => {
179 + for value in values.iter().filter_map(Value::as_str) {
180 + push_schema_type(&mut types, value);
181 + }
182 + }
183 + _ => {}
184 + }
185 + if types.is_empty()
186 + && let Some(value) = schema.get("const")
187 + {
188 + push_value_type(&mut types, value);
189 + }
190 + if types.is_empty()
191 + && let Some(values) = schema.get("enum").and_then(Value::as_array)
192 + {
193 + for value in values {
194 + push_value_type(&mut types, value);
195 + }
196 + }
197 + if types.is_empty() {
198 + XTML_TYPES.to_vec()
199 + } else {
200 + types
201 + }
202 +}
203 +
204 +fn push_value_type(types: &mut Vec<&'static str>, value: &Value) {
205 + let xtml_type = match value {
206 + Value::String(_) => "string",
207 + Value::Number(_) => "number",
208 + Value::Bool(_) => "boolean",
209 + Value::Null => "null",
210 + Value::Object(_) => "object",
211 + Value::Array(_) => "array",
212 + };
213 + if !types.contains(&xtml_type) {
214 + types.push(xtml_type);
215 + }
216 +}
217 +
218 +fn narrow_schema_type(schema: &Value, xtml_type: &str) -> Value {
219 + let Some(mut schema) = schema.as_object().cloned() else {
220 + return schema.clone();
221 + };
222 + let json_type = if xtml_type == "number" && explicitly_integer_only(&schema) {
223 + "integer"
224 + } else {
225 + xtml_type
226 + };
227 + schema.insert("type".to_string(), Value::String(json_type.to_string()));
228 + Value::Object(schema)
229 +}
230 +
231 +fn explicitly_integer_only(schema: &Map<String, Value>) -> bool {
232 + match schema.get("type") {
233 + Some(Value::String(value)) => value == "integer",
234 + Some(Value::Array(values)) => {
235 + let values = values.iter().filter_map(Value::as_str).collect::<Vec<_>>();
236 + values.contains(&"integer") && !values.contains(&"number")
237 + }
238 + _ => false,
239 + }
240 +}
241 +
242 +fn push_schema_type(types: &mut Vec<&'static str>, json_type: &str) {
243 + let xtml_type = match json_type {
244 + "string" => Some("string"),
245 + "integer" | "number" => Some("number"),
246 + "boolean" => Some("boolean"),
247 + "null" => Some("null"),
248 + "object" => Some("object"),
249 + "array" => Some("array"),
250 + _ => None,
251 + };
252 + if let Some(xtml_type) = xtml_type
253 + && !types.contains(&xtml_type)
254 + {
255 + types.push(xtml_type);
256 + }
257 +}
258 +
259 +fn string_argument_content(schema: &Value) -> Format {
260 + let Some(schema) = schema.as_object() else {
261 + return Format::any_text_excluding(&[ARG_CLOSE, CALL_CLOSE]);
262 + };
263 + let values = schema
264 + .get("enum")
265 + .and_then(Value::as_array)
266 + .cloned()
267 + .or_else(|| schema.get("const").cloned().map(|value| vec![value]));
268 + let Some(values) = values else {
269 + return Format::any_text_excluding(&[ARG_CLOSE, CALL_CLOSE]);
270 + };
271 + if values.is_empty()
272 + || values.len() > 256
273 + || values
274 + .iter()
275 + .any(|value| value.as_str().is_none_or(|value| value.contains("<|")))
276 + {
277 + return Format::any_text_excluding(&[ARG_CLOSE, CALL_CLOSE]);
278 + }
279 + let values = values.iter().filter_map(Value::as_str).collect::<Vec<_>>();
280 + match values.as_slice() {
281 + [value] => Format::const_string(*value),
282 + _ => Format::or(values.into_iter().map(Format::const_string).collect()),
283 + }
284 +}
285 +
286 +fn raw_json_arguments(parameters: &Value, options: StructuralTagOptions) -> Format {
287 + Format::tag(
288 + format!("{JSON_OPEN} type=\"object\"{SEP}"),
289 + json_schema(parameters.clone(), options),
290 + JSON_CLOSE,
291 + )
292 +}
293 +
294 +fn permissive_argument() -> Format {
295 + let key = Format::regex(r#"(?:[^<\"&]|&(?:amp|quot);|<[^|])*"#);
296 + let alternatives = XTML_TYPES
297 + .iter()
298 + .map(|xtml_type| {
299 + Format::sequence(vec![
300 + key.clone(),
301 + Format::const_string(format!("\" type=\"{xtml_type}\"{SEP}")),
302 + if *xtml_type == "string" {
303 + Format::any_text_excluding(&[ARG_CLOSE, CALL_CLOSE])
304 + } else {
305 + Format::json_schema(Value::Bool(true))
306 + },
307 + ])
308 + })
309 + .collect();
310 + Format::tag(
311 + format!("{OPEN}argument key=\""),
312 + Format::or(alternatives),
313 + ARG_CLOSE,
314 + )
315 +}
316 +
317 +fn json_schema(schema: Value, options: StructuralTagOptions) -> Format {
318 + Format::JsonSchema(
319 + JsonSchemaFormat::new(schema)
320 + .with_any_order(options.any_order)
321 + .with_max_whitespace_cnt(options.max_whitespace_cnt),
322 + )
323 +}
324 +
325 +fn root_definitions(schema: &Map<String, Value>) -> Map<String, Value> {
326 + ["$defs", "definitions"]
327 + .into_iter()
328 + .filter_map(|key| schema.get(key).map(|value| (key.to_string(), value.clone())))
329 + .collect()
330 +}
331 +
332 +fn attach_root_definitions(schema: &Value, root_defs: &Map<String, Value>) -> Value {
333 + let Some(mut schema) = schema.as_object().cloned() else {
334 + return schema.clone();
335 + };
336 + for (key, value) in root_defs {
337 + schema.entry(key.clone()).or_insert_with(|| value.clone());
338 + }
339 + Value::Object(schema)
340 +}
341 +
342 +fn escape_attr_value(value: &str) -> String {
343 + value.replace('&', "&amp;").replace('"', "&quot;")
344 +}
345 +
346 +#[cfg(test)]
347 +mod tests {
348 + use expect_test::expect;
349 + use serde_json::json;
350 + use xgrammar_structural_tag::builders::StructuralTagOptions;
351 + use xgrammar_structural_tag::{
352 + FunctionDefinition, FunctionToolParam, ToolChoice, ToolParam, build_structural_tag,
353 + };
354 +
355 + use super::KimiK3StructuralTagBuilder;
356 +
357 + fn tool(name: &str, parameters: serde_json::Value) -> ToolParam {
358 + ToolParam::Function(FunctionToolParam::new(
359 + FunctionDefinition::new(name).with_parameters(parameters),
360 + ))
361 + }
362 +
363 + #[test]
364 + fn required_structural_tag_matches_xtml_channels() {
365 + let tools = vec![tool(
366 + "get_weather",
367 + json!({
368 + "$defs": {
369 + "place": { "type": "object", "properties": { "city": { "type": "string" } } }
370 + },
371 + "type": "object",
372 + "properties": {
373 + "unit": { "type": "string", "enum": ["celsius", "fahrenheit"] },
374 + "place": { "$ref": "#/$defs/place", "type": "object" }
375 + },
376 + "required": ["place"]
377 + }),
378 + )];
379 + let tag = build_structural_tag(
380 + KimiK3StructuralTagBuilder,
381 + &tools,
382 + ToolChoice::required(),
383 + StructuralTagOptions::default().with_reasoning(false),
384 + )
385 + .unwrap();
386 +
387 + expect![[r##"{"type":"structural_tag","format":{"type":"sequence","elements":[{"type":"optional","content":{"type":"const_string","value":"<|open|>response<|sep|>"}},{"type":"tag","begin":"","content":{"type":"any_text","excludes":["<|close|>response<|sep|>","<|open|>tools<|sep|>","<|close|>message<|sep|>","<|end_of_msg|>"]},"end":"<|close|>response<|sep|>"},{"type":"tag","begin":"<|open|>tools<|sep|>","content":{"type":"tags_with_separator","tags":[{"begin":"<|open|>call tool=\"get_weather\" index=\"","content":{"type":"sequence","elements":[{"type":"regex","pattern":"[1-9][0-9]*"},{"type":"const_string","value":"\"<|sep|>"},{"type":"or","elements":[{"type":"plus","content":{"type":"or","elements":[{"type":"tag","begin":"<|open|>argument key=\"unit\" type=\"string\"<|sep|>","content":{"type":"or","elements":[{"type":"const_string","value":"celsius"},{"type":"const_string","value":"fahrenheit"}]},"end":"<|close|>argument<|sep|>"},{"type":"tag","begin":"<|open|>argument key=\"place\" type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$ref":"#/$defs/place","type":"object","$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}}},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>argument<|sep|>"}]}},{"type":"tag","begin":"<|open|>json type=\"object\"<|sep|>","content":{"type":"json_schema","json_schema":{"$defs":{"place":{"type":"object","properties":{"city":{"type":"string"}}}},"type":"object","properties":{"unit":{"type":"string","enum":["celsius","fahrenheit"]},"place":{"$ref":"#/$defs/place","type":"object"}},"required":["place"]},"style":"json","any_order":false,"max_whitespace_cnt":null},"end":"<|close|>json<|sep|>"}]}]},"end":"<|close|>call<|sep|>"}],"separator":"","at_least_one":true,"stop_after_first":false},"end":"<|close|>tools<|sep|>"},{"type":"optional","content":{"type":"const_string","value":"<|close|>message<|sep|>"}}]}}"##]].assert_eq(&tag.to_json_string().unwrap());
388 + }
389 +
390 + #[test]
391 + fn typed_arguments_require_one_tag_only_for_nonempty_required() {
392 + let required = super::typed_arguments(
393 + &json!({
394 + "type": "object",
395 + "properties": { "query": { "type": "string" } },
396 + "required": ["query"]
397 + }),
398 + StructuralTagOptions::default(),
399 + );
400 + let optional = super::typed_arguments(
401 + &json!({
402 + "type": "object",
403 + "properties": { "query": { "type": "string" } }
404 + }),
405 + StructuralTagOptions::default(),
406 + );
407 + let empty_required = super::typed_arguments(
408 + &json!({
409 + "type": "object",
410 + "properties": { "query": { "type": "string" } },
411 + "required": []
412 + }),
413 + StructuralTagOptions::default(),
414 + );
415 +
416 + assert_eq!(serde_json::to_value(required).unwrap()["type"], "plus");
417 + assert_eq!(serde_json::to_value(optional).unwrap()["type"], "star");
418 + assert_eq!(
419 + serde_json::to_value(empty_required).unwrap()["type"],
420 + "star"
421 + );
422 + }
423 +
424 + #[test]
425 + fn reasoning_grammar_starts_inside_prefilled_think_channel() {
426 + let tools = vec![tool("ping", json!({ "type": "object", "properties": {} }))];
427 + let tag = build_structural_tag(
428 + KimiK3StructuralTagBuilder,
429 + &tools,
430 + ToolChoice::auto(),
431 + StructuralTagOptions::default().with_reasoning(true),
432 + )
433 + .unwrap();
434 + let value = serde_json::to_value(tag).unwrap();
435 +
436 + assert_eq!(
437 + value["format"]["elements"][0]["end"],
438 + "<|close|>think<|sep|>"
439 + );
440 + assert_eq!(
441 + value["format"]["elements"][1]["value"],
442 + "<|open|>response<|sep|>"
443 + );
444 + assert_eq!(value["format"]["elements"][3]["type"], "optional");
445 + }
446 +
447 + #[test]
448 + fn forced_choice_keeps_only_the_named_tool() {
449 + let tools = vec![
450 + tool("search", json!({ "type": "object" })),
451 + tool("lookup", json!({ "type": "object" })),
452 + ];
453 + let tag = build_structural_tag(
454 + KimiK3StructuralTagBuilder,
455 + &tools,
456 + ToolChoice::function("lookup"),
457 + StructuralTagOptions::default().with_reasoning(false),
458 + )
459 + .unwrap()
460 + .to_json_string()
461 + .unwrap();
462 +
463 + assert!(tag.contains("lookup"));
464 + assert!(!tag.contains("search"));
465 + }
466 +
467 + #[test]
468 + fn union_argument_content_matches_its_xtml_type() {
469 + let tools = vec![tool(
470 + "set_count",
471 + json!({
472 + "type": "object",
473 + "properties": {
474 + "count": { "type": ["integer", "null"] }
475 + }
476 + }),
477 + )];
478 + let tag = build_structural_tag(
479 + KimiK3StructuralTagBuilder,
480 + &tools,
481 + ToolChoice::required(),
482 + StructuralTagOptions::default(),
483 + )
484 + .unwrap()
485 + .to_json_string()
486 + .unwrap();
487 +
488 + assert!(tag.contains(r#"type=\"number\""#));
489 + assert!(tag.contains(r#""json_schema":{"type":"integer"}"#), "{tag}");
490 + assert!(tag.contains(r#"type=\"null\""#));
491 + assert!(tag.contains(r#""json_schema":{"type":"null"}"#), "{tag}");
492 + }
493 +
494 + #[test]
495 + fn unsafe_string_enum_falls_back_as_a_whole() {
496 + let format = super::string_argument_content(&json!({
497 + "type": "string",
498 + "enum": ["safe", "<|unsafe"]
499 + }));
500 +
501 + assert_eq!(serde_json::to_value(format).unwrap()["type"], "any_text");
502 + }
503 +}
rust/src/parser/src/unified/mod.rs
+2
@@ -6,10 +6,12 @@
6 mod combined;
7 mod gemma4;
8 mod inkling;
9 +mod kimi_k3;
10
11 pub use combined::CombinedParser;
12 pub use gemma4::Gemma4UnifiedParser;
13 pub use inkling::InklingUnifiedParser;
14 +pub use kimi_k3::{KimiK3StructuralTagBuilder, KimiK3UnifiedParser};
15 use thiserror::Error;
16 use thiserror_ext::Macro;
17 use vllm_tokenizer::DynTokenizer;
rust/src/server/src/routes/openai/chat_completions/convert.rs
+70 -3
@@ -89,6 +89,15 @@ pub(super) fn prepare_chat_request(
89 )?;
90
91 let template_kwargs = request.chat_template_kwargs.unwrap_or_default();
92 + let response_format =
93 + request.response_format.as_ref().map(serde_json::to_value).transpose().map_err(
94 + |error| {
95 + ApiError::invalid_request(
96 + format!("failed to serialize response_format: {error}"),
97 + Some("response_format"),
98 + )
99 + },
100 + )?;
101
102 let include_usage = (request.stream_options.as_ref())
103 .and_then(|options| options.include_usage)
@@ -150,6 +159,7 @@ pub(super) fn prepare_chat_request(
159 generation_prompt_mode,
160 chat_template: request.chat_template,
161 reasoning_effort: request.reasoning_effort,
162 + response_format,
163 template_kwargs,
164 },
165 tools: convert_tools(request.tools)?,
@@ -400,6 +410,7 @@ fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoic
410 #[cfg(test)]
411 mod tests {
412 use std::collections::HashMap;
413 + use std::sync::Arc;
414
415 use axum::http::HeaderMap;
416 use expect_test::expect;
@@ -407,16 +418,18 @@ mod tests {
418 use serde_json::json;
419 use vllm_chat::{
420 AssistantContentBlock, AssistantToolCall, ChatContentPart, ChatMessage as VllmChatMessage,
410 - ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
411 - SamplingParams as VllmSamplingParams,
421 + ChatRenderer, ChatTool as VllmChatTool, ChatToolChoice, GenerationPromptMode,
422 + KimiK3ChatRenderer, SamplingParams as VllmSamplingParams,
423 };
413 - use vllm_text::output::TextDecodeOptions;
424 + use vllm_text::{Prompt, output::TextDecodeOptions};
425 + use vllm_tokenizer::{Tokenizer, test_utils::TestTokenizer};
426
427 use super::prepare_chat_request;
428 use crate::lora::LoraModelResolution;
429 use crate::routes::openai::chat_completions::types::{
430 AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
431 };
432 + use crate::routes::openai::utils::structured_outputs::{JsonSchemaFormat, ResponseFormat};
433 use crate::routes::openai::utils::types::{
434 AudioUrl, ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, InputAudio,
435 MessageContent, StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
@@ -473,6 +486,60 @@ mod tests {
486 assert!(prepared.chat_request.parallel_tool_calls);
487 }
488
489 + #[test]
490 + fn prepare_chat_request_passes_response_format_to_kimi_k3_renderer() {
491 + let mut request = base_request();
492 + request.model = "moonshotai/Kimi-K3".to_string();
493 + let response_format = ResponseFormat::JsonSchema {
494 + json_schema: JsonSchemaFormat {
495 + name: "answer".to_string(),
496 + description: None,
497 + schema: json!({
498 + "type": "object",
499 + "properties": {
500 + "answer": {"type": "string"}
501 + },
502 + "required": ["answer"]
503 + }),
504 + strict: Some(true),
505 + },
506 + };
507 + request.response_format = Some(response_format.clone());
508 + let template_response_format = json!({"type": "json_object"});
509 + request.chat_template_kwargs = Some(HashMap::from([(
510 + "response_format".to_string(),
511 + template_response_format.clone(),
512 + )]));
513 +
514 + let prepared = prepare_chat_request(
515 + request,
516 + &served(&["moonshotai/Kimi-K3"]),
517 + ResolvedRequestContext::default(),
518 + )
519 + .expect("request is valid");
520 +
521 + assert_eq!(
522 + prepared.chat_request.chat_options.response_format.as_ref(),
523 + Some(&serde_json::to_value(response_format).unwrap())
524 + );
525 + assert_eq!(
526 + prepared.chat_request.chat_options.template_kwargs.get("response_format"),
527 + Some(&template_response_format)
528 + );
529 +
530 + let tokenizer = Arc::new(TestTokenizer::new());
531 + let prompt = KimiK3ChatRenderer::new(tokenizer.clone())
532 + .render(&prepared.chat_request)
533 + .expect("Kimi K3 rendering succeeds")
534 + .prompt;
535 + let Prompt::TokenIds(prompt_token_ids) = prompt else {
536 + panic!("Kimi K3 renders token IDs");
537 + };
538 + let prompt = tokenizer.decode(&prompt_token_ids, false).expect("Kimi K3 prompt decodes");
539 + assert!(prompt.contains("response_format=json_schema"));
540 + assert!(prompt.contains(r#""answer":{"type":"string"}"#));
541 + }
542 +
543 #[test]
544 fn prepare_chat_request_maps_text_parts() {
545 let mut request = base_request();
rust/src/server/src/routes/tokenize/types.rs
+1
@@ -82,6 +82,7 @@ impl TokenizeChatRequest {
82 generation_prompt_mode,
83 chat_template: self.chat_template,
84 reasoning_effort: None,
85 + response_format: None,
86 template_kwargs: self.chat_template_kwargs.unwrap_or_default(),
87 },
88 tools: convert_tools(self.tools)?,
rust/src/tokenizer/src/tiktoken.rs
+3 -1
@@ -532,7 +532,7 @@ fn detect_bpe_pattern(config: &TiktokenModelConfig) -> &'static str {
532 let model_type = config.effective_model_type();
533
534 match model_type {
535 - Some("kimi" | "kimi_k2" | "kimi_k25" | "deepseek_v3") => KIMI_PATTERN,
535 + Some("kimi" | "kimi_k2" | "kimi_k25" | "kimi_k3" | "deepseek_v3") => KIMI_PATTERN,
536 _ => CL100K_BASE_PATTERN,
537 }
538 }
@@ -817,6 +817,7 @@ mod tests {
817 #[test]
818 fn tiktoken_detects_kimi_pattern_from_model_type() {
819 let kimi = config_json!({ "model_type": "kimi_k25" });
820 + let kimi_k3 = config_json!({ "model_type": "kimi_k3" });
821 let baseten_kimi = config_json!({ "model_type": "deepseek_v3" });
822 let nested_kimi = config_json!({
823 "model_type": "composite_wrapper",
@@ -830,6 +831,7 @@ mod tests {
831 let missing = config_json!({ "text_config": {} });
832
833 assert_eq!(detect_bpe_pattern(&kimi), KIMI_PATTERN);
834 + assert_eq!(detect_bpe_pattern(&kimi_k3), KIMI_PATTERN);
835 assert_eq!(detect_bpe_pattern(&baseten_kimi), KIMI_PATTERN);
836 assert_eq!(detect_bpe_pattern(&nested_kimi), CL100K_BASE_PATTERN);
837 assert_eq!(detect_bpe_pattern(&generic), CL100K_BASE_PATTERN);