@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0b68f21e7c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 2, 19:30 UTC

[Rust Frontend] Add reasoning/tool parser & renderer roundtrip tests (#43582)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed May 27, 2026 at 08:49 UTC 0b68f21e7c6ff146f12fd9a9018f5052d4b4722e
11 files changed +671 -43
rust/Cargo.lock
+3
@@ -2591,6 +2591,7 @@ version = "2.18.0"
2591 source = "registry+https://github.com/rust-lang/crates.io-index"
2592 checksum = "328251e58ad8e415be6198888fc207502727dc77945806421ab34f35bf012e7d"
2593 dependencies = [
2594 + "indexmap 2.13.0",
2595 "memo-map",
2596 "serde",
2597 "serde_json",
@@ -5608,6 +5609,7 @@ dependencies = [
5609 "minijinja",
5610 "minijinja-contrib",
5611 "openai-harmony",
5612 + "paste",
5613 "reqwest",
5614 "rmp-serde",
5615 "serde",
@@ -5810,6 +5812,7 @@ dependencies = [
5812 "serde",
5813 "serde_json",
5814 "serde_with",
5815 + "serial_test",
5816 "tempfile",
5817 "thiserror 2.0.18",
5818 "thiserror-ext",
rust/Cargo.toml
+4 -3
@@ -45,13 +45,14 @@ http-body = "1.0.1"
45 itertools = "0.14.0"
46 libc = "0.2.177"
47 llm-multimodal = { git = "https://github.com/vllm-project/llm-multimodal", rev = "5b558989844d1c7af3e43d0f604069ffd9c06320" }
48 -minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls"] }
48 +minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
49 minijinja-contrib = { version = "2.0", features = ["pycompat"] }
50 native-tls-vendored = { package = "native-tls", version = "0.2.18", features = ["vendored"] }
51 ndarray = { version = "0.16.1", features = ["serde"] }
52 openai-harmony = "0.0.8"
53 openai-protocol = "1.6.0"
54 parking_lot = "0.12.5"
55 +paste = "1.0.15"
56 prometheus-client = "0.24.0"
57 prometheus-client-derive-encode = "0.5.0"
58 prost = "0.14.3"
@@ -65,11 +66,11 @@ rustc-hash = "1.1.0"
66 serde = { version = "1.0.228", features = ["derive"] }
67 serde-json-fmt = "0.1.0"
68 serde_default = "0.2.0"
68 -serde_json = "1.0.145"
69 +serde_json = { version = "1.0.145", features = ["arbitrary_precision", "preserve_order"] }
70 serde_repr = "0.1.20"
71 serde_tuple = "1.1.3"
72 serde_with = "3.18.0"
72 -serial_test = "3.2.0"
73 +serial_test = { version = "3.2.0", features = ["file_locks"] }
74 socket2 = "0.6.3"
75 subenum = "1.1.3"
76 task-local = "0.1.1"
rust/src/chat/Cargo.toml
+2 -1
@@ -39,8 +39,9 @@ anyhow.workspace = true
39 bytes.workspace = true
40 clap.workspace = true
41 expect-test.workspace = true
42 +paste.workspace = true
43 rmp-serde.workspace = true
43 -serial_test = { workspace = true, features = ["file_locks"] }
44 +serial_test.workspace = true
45 tempfile.workspace = true
46 tokio.workspace = true
47 tracing-subscriber.workspace = true
rust/src/chat/src/event.rs
+26
@@ -53,6 +53,25 @@ impl AssistantContentBlock {
53 _ => None,
54 }
55 }
56 +
57 + /// Return a copy of this block with leading and trailing whitespace trimmed from all text
58 + /// fields and tool call arguments, or `None` if the resulting text would be empty.
59 + pub fn trim(mut self) -> Option<Self> {
60 + match &mut self {
61 + Self::Text { text } | Self::Reasoning { text } => {
62 + let trimmed_text = text.trim();
63 + if trimmed_text.is_empty() {
64 + return None;
65 + } else {
66 + *text = trimmed_text.to_string();
67 + }
68 + }
69 + Self::ToolCall(call) => {
70 + call.arguments = call.arguments.trim().to_string();
71 + }
72 + }
73 + Some(self)
74 + }
75 }
76
77 #[easy_ext::ext(AssistantMessageExt)]
@@ -119,6 +138,13 @@ impl AssistantMessage {
138 pub(crate) fn push_block(&mut self, block: AssistantContentBlock) {
139 self.content.push(block);
140 }
141 +
142 + /// Return a copy of this message with leading and trailing whitespace trimmed from all text
143 + /// fields and tool call arguments, and with any blocks that are empty after trimming removed.
144 + pub fn trim(mut self) -> Self {
145 + self.content = self.content.into_iter().filter_map(|block| block.trim()).collect();
146 + self
147 + }
148 }
149
150 /// Streamed chat event emitted by [`crate::ChatEventStream`].
rust/src/chat/src/renderer/hf/tojson.rs
+12 -11
@@ -1,7 +1,7 @@
1 use minijinja::value::{Kwargs, ViaDeserialize};
2 use minijinja::{Error as MinijinjaError, ErrorKind, Value};
3 use serde::Deserialize;
4 -use serde_json::{self, Value as JsonValue};
4 +use serde_json::Value as JsonValue;
5 use serde_json_fmt::{JsonFormat, JsonSyntaxError};
6 use thiserror_ext::AsReport;
7
@@ -13,7 +13,7 @@ use thiserror_ext::AsReport;
13 /// - extra kwargs such as `ensure_ascii`, `separators`, and `sort_keys`
14 /// - Python-style `indent` handling
15 pub(super) fn hf_tojson_filter(
16 - value: Value,
16 + ViaDeserialize(value): ViaDeserialize<JsonValue>,
17 kwargs: Kwargs,
18 ) -> std::result::Result<Value, MinijinjaError> {
19 let ensure_ascii = kwargs.get::<Option<bool>>("ensure_ascii")?.unwrap_or(false);
@@ -30,18 +30,11 @@ pub(super) fn hf_tojson_filter(
30
31 kwargs.assert_all_used()?;
32
33 - let json_value: serde_json::Value = serde_json::to_value(&value).map_err(|e| {
34 - MinijinjaError::new(
35 - ErrorKind::InvalidOperation,
36 - format!("Failed to convert to JSON value: {e}"),
37 - )
38 - })?;
39 -
33 let json_str = {
34 let value_to_serialize = if sort_keys {
42 - &sort_json_keys(&json_value)
35 + &sort_json_keys(&value)
36 } else {
44 - &json_value
37 + &value
38 };
39
40 build_json_format(indent, separators.0, separators.1, ensure_ascii)?
@@ -214,6 +207,14 @@ mod tests {
207 assert_eq!(rendered, "{\"x\":[1,2]}");
208 }
209
210 + #[test]
211 + fn tojson_preserves_arbitrary_precision_number_spelling() {
212 + let payload = serde_json::from_str(r#"{"x":2,"y":1.00}"#).unwrap();
213 + let rendered = render("{{ payload|tojson }}", payload);
214 +
215 + assert_eq!(rendered, "{\"x\": 2, \"y\": 1.00}");
216 + }
217 +
218 #[test]
219 fn tojson_supports_negative_indent_as_newline_only() {
220 let rendered = render("{{ payload|tojson(indent=-1) }}", json!([1, 2]));
rust/src/chat/tests/roundtrip.rs new
+541
@@ -0,0 +1,541 @@
1 +//! Text-level roundtrip tests for the real chat-template and output-processor pairing.
2 +//!
3 +//! The invariant under test is that a structured assistant message rendered as history can be
4 +//! parsed from the generated assistant completion and then rendered back to the exact same
5 +//! assistant-completion text.
6 +
7 +use std::pin::Pin;
8 +use std::sync::Arc;
9 +
10 +use anyhow::{Context as _, Result, bail, ensure};
11 +use futures::{Stream, StreamExt as _, stream};
12 +use serde_json_fmt::JsonFormat as JsonFmt;
13 +use serial_test::file_serial;
14 +use vllm_chat::{
15 + AssistantContentBlock, AssistantMessage, AssistantMessageExt as _, AssistantToolCall,
16 + ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice, FinishReason,
17 + GenerationPromptMode, LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection,
18 + RendererSelection, load_model_backends,
19 +};
20 +use vllm_text::{DecodedTextEvent, Finished, Prompt};
21 +
22 +/// One model/parser configuration used to run the fixed roundtrip fixtures.
23 +struct RoundtripCase {
24 + /// Hugging Face model id resolved through the production backend loader.
25 + model_id: &'static str,
26 + /// Final assistant-history suffix rendered by the chat template but not
27 + /// generated by the model body consumed by the output processor.
28 + // TODO: we should adopt `ContinueFinalAssistant` mode to naturally handle this.
29 + assistant_stop_suffix: &'static str,
30 + /// Tool parser selection used by the output processor.
31 + tool_call_parser: ParserSelection,
32 + /// Reasoning parser selection used by the output processor.
33 + reasoning_parser: ParserSelection,
34 + /// JSON formatting expected after this model's template has materialized
35 + /// tool-call arguments.
36 + json_fmt: JsonFmt,
37 +}
38 +
39 +impl RoundtripCase {
40 + /// Qwen3 XML tool-call format with `qwen3` reasoning tags.
41 + fn qwen3() -> Self {
42 + Self {
43 + model_id: "Qwen/Qwen3-0.6B",
44 + assistant_stop_suffix: "<|im_end|>\n",
45 + tool_call_parser: ParserSelection::Auto,
46 + reasoning_parser: ParserSelection::Auto,
47 + json_fmt: spaced_json_fmt(),
48 + }
49 + }
50 +
51 + /// Qwen3.5 coder-style JSON tool-call format with `qwen3` reasoning tags.
52 + fn qwen35() -> Self {
53 + Self {
54 + model_id: "Qwen/Qwen3.5-4B",
55 + assistant_stop_suffix: "<|im_end|>\n",
56 + tool_call_parser: ParserSelection::Auto,
57 + reasoning_parser: ParserSelection::Auto,
58 + json_fmt: compact_json_fmt(),
59 + }
60 + }
61 +
62 + /// MiniMax M2.5 XML invoke format with `<think>` reasoning tags.
63 + fn minimax_m25() -> Self {
64 + Self {
65 + model_id: "MiniMaxAI/MiniMax-M2.5",
66 + assistant_stop_suffix: "[e~[\n",
67 + tool_call_parser: ParserSelection::Auto,
68 + reasoning_parser: ParserSelection::Auto,
69 + json_fmt: compact_json_fmt(),
70 + }
71 + }
72 +
73 + /// DeepSeek V4 DSML tool-call format.
74 + fn deepseek_v4() -> Self {
75 + Self {
76 + model_id: "deepseek-ai/DeepSeek-V4-Flash",
77 + assistant_stop_suffix: "<｜end▁of▁sentence｜>",
78 + tool_call_parser: ParserSelection::Auto,
79 + reasoning_parser: ParserSelection::Auto,
80 + json_fmt: compact_json_fmt(),
81 + }
82 + }
83 +
84 + /// GLM-4.7 XML-like argument format with `<think>` reasoning tags.
85 + fn glm47() -> Self {
86 + Self {
87 + model_id: "zai-org/GLM-4.7-Flash",
88 + assistant_stop_suffix: "",
89 + tool_call_parser: ParserSelection::Auto,
90 + reasoning_parser: ParserSelection::Auto,
91 + json_fmt: compact_json_fmt(),
92 + }
93 + }
94 +
95 + /// Kimi K2.5 tool-call format with `<think>` reasoning tags.
96 + #[allow(dead_code)]
97 + fn kimi_k25() -> Self {
98 + Self {
99 + model_id: "moonshotai/Kimi-K2.5",
100 + assistant_stop_suffix: "<|im_end|>",
101 + tool_call_parser: ParserSelection::Auto,
102 + reasoning_parser: ParserSelection::Auto,
103 + json_fmt: spaced_json_fmt(),
104 + }
105 + }
106 +}
107 +
108 +macro_rules! roundtrip_tests {
109 + ($($case:ident => [$($fixture:ident),* $(,)?]),+ $(,)?) => {
110 + paste::paste! {
111 + $(
112 + $(
113 + #[tokio::test]
114 + #[file_serial([<hf_ $case>])]
115 + async fn [<roundtrip_ $case _ $fixture>]() -> Result<()> {
116 + [<run_roundtrip_ $fixture>](RoundtripCase::$case()).await
117 + }
118 + )*
119 + )+
120 + }
121 + };
122 +}
123 +
124 +roundtrip_tests! {
125 + qwen3 => [reasoning_and_content, tool_call_mix],
126 + qwen35 => [reasoning_and_content, tool_call_mix],
127 + minimax_m25 => [reasoning_and_content, tool_call_mix],
128 + deepseek_v4 => [reasoning_and_content, tool_call_mix],
129 + glm47 => [reasoning_and_content, tool_call_mix],
130 +
131 + // Note: Kimi K2.5 strips the reasoning content in history.
132 + // TODO: we don't respect model-generated tool call id now so `tool_call_mix` cannot pass.
133 + // kimi_k25 => [tool_call_mix],
134 +}
135 +
136 +/// Run the fixed reasoning+content fixture for one model/parser case.
137 +async fn run_roundtrip_reasoning_and_content(case: RoundtripCase) -> Result<()> {
138 + let backends = load_roundtrip_backends(&case).await?;
139 + let request = roundtrip_request(
140 + "roundtrip-reasoning-content",
141 + vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")],
142 + Vec::new(),
143 + );
144 + let expected_reasoning = "Need compute 2 + 2 directly.";
145 + let expected_text = "The answer is 4.";
146 +
147 + let result = run_roundtrip(
148 + &case,
149 + &backends,
150 + &request,
151 + AssistantMessage {
152 + content: vec![
153 + AssistantContentBlock::Reasoning {
154 + text: expected_reasoning.to_string(),
155 + },
156 + AssistantContentBlock::Text {
157 + text: expected_text.to_string(),
158 + },
159 + ],
160 + },
161 + )
162 + .await?;
163 +
164 + assert_eq!(
165 + result.parsed_message.reasoning().as_deref().map(str::trim),
166 + Some(expected_reasoning)
167 + );
168 + assert_eq!(result.parsed_message.text().trim(), expected_text);
169 + assert_eq!(result.parsed_message.tool_calls().count(), 0);
170 +
171 + assert_eq!(
172 + result.rerendered_closed_completion,
173 + result.closed_completion
174 + );
175 +
176 + Ok(())
177 +}
178 +
179 +/// Run the fixed reasoning+multiple-tools fixture for one model/parser case.
180 +async fn run_roundtrip_tool_call_mix(case: RoundtripCase) -> Result<()> {
181 + let backends = load_roundtrip_backends(&case).await?;
182 + let request = roundtrip_request(
183 + "roundtrip-reasoning-tools",
184 + vec![ChatMessage::text(
185 + ChatRole::User,
186 + "Check Shanghai weather and add 1.00 plus 2.",
187 + )],
188 + test_tools(),
189 + );
190 + let expected_reasoning = "Need call the weather and add tools.";
191 + let expected_text = "I will call the tools.";
192 +
193 + let result = run_roundtrip(
194 + &case,
195 + &backends,
196 + &request,
197 + AssistantMessage {
198 + content: vec![
199 + AssistantContentBlock::Reasoning {
200 + text: expected_reasoning.to_string(),
201 + },
202 + AssistantContentBlock::Text {
203 + text: expected_text.to_string(),
204 + },
205 + AssistantContentBlock::ToolCall(AssistantToolCall {
206 + id: "functions.get_weather:0".to_string(),
207 + name: "get_weather".to_string(),
208 + arguments: r#"{"location":"Shanghai"}"#.to_string(),
209 + }),
210 + AssistantContentBlock::ToolCall(AssistantToolCall {
211 + id: "functions.add:1".to_string(),
212 + name: "add".to_string(),
213 + // Intentionally use a non-lexical order of keys and a different number
214 + // formatting style to verify text-level fidelity of the roundtrip.
215 + arguments: r#"{"y":1.00,"x":2}"#.to_string(),
216 + }),
217 + ],
218 + },
219 + )
220 + .await?;
221 +
222 + assert_eq!(
223 + result.parsed_message.reasoning().as_deref().map(str::trim),
224 + Some(expected_reasoning)
225 + );
226 + assert_eq!(result.parsed_message.text().trim(), expected_text);
227 +
228 + let tool_calls = result.parsed_message.tool_calls().collect::<Vec<_>>();
229 + assert_eq!(
230 + tool_calls.len(),
231 + 2,
232 + "parsed message: {:#?}",
233 + result.parsed_message
234 + );
235 + assert_eq!(tool_calls[0].name, "get_weather");
236 + assert_eq!(
237 + tool_calls[0].arguments,
238 + expected_arguments(&case, r#"{"location": "Shanghai"}"#)?,
239 + );
240 + assert_eq!(tool_calls[1].name, "add");
241 + assert_eq!(
242 + tool_calls[1].arguments,
243 + expected_arguments(&case, r#"{"y": 1.00, "x": 2}"#)?,
244 + );
245 +
246 + assert_eq!(
247 + result.rerendered_closed_completion,
248 + result.closed_completion
249 + );
250 +
251 + Ok(())
252 +}
253 +
254 +/// Compact JSON argument formatting used by JSON-native parsers/renderers.
255 +fn compact_json_fmt() -> JsonFmt {
256 + JsonFmt::new()
257 +}
258 +
259 +/// Python `json.dumps`-style compact formatting with a space after commas and
260 +/// colons.
261 +fn spaced_json_fmt() -> JsonFmt {
262 + JsonFmt::new()
263 + .comma(", ")
264 + .expect("literal comma separator is valid JSON")
265 + .colon(": ")
266 + .expect("literal colon separator is valid JSON")
267 +}
268 +
269 +/// Parse and format expected tool-call arguments from raw JSON text.
270 +/// Pass in a raw JSON string instead of a structured value to ensure the exact precision and
271 +/// formatting of numbers are preserved.
272 +fn expected_arguments(case: &RoundtripCase, raw_json: &str) -> Result<String> {
273 + let value: serde_json::Value =
274 + serde_json::from_str(raw_json).context("invalid expected tool-call arguments")?;
275 +
276 + case.json_fmt
277 + .format_to_string(&value)
278 + .context("failed to format expected tool-call arguments")
279 +}
280 +
281 +/// Load the real model chat/text backend for one roundtrip case.
282 +async fn load_roundtrip_backends(case: &RoundtripCase) -> Result<vllm_chat::LoadedModelBackends> {
283 + load_model_backends(
284 + case.model_id,
285 + LoadModelBackendsOptions {
286 + renderer: RendererSelection::Auto,
287 + ..Default::default()
288 + },
289 + )
290 + .await
291 + .with_context(|| format!("failed to load HF model files for {}", case.model_id))
292 +}
293 +
294 +/// Roundtrip artifacts needed for semantic and exact-text assertions.
295 +struct RoundtripResult {
296 + /// Final assistant message reconstructed by the output processor.
297 + parsed_message: AssistantMessage,
298 + /// Assistant-completion suffix cut from rendering the expected assistant as
299 + /// history.
300 + closed_completion: String,
301 + /// Assistant-completion suffix cut after rendering the parsed assistant
302 + /// back as history.
303 + rerendered_closed_completion: String,
304 +}
305 +
306 +/// Render, parse, and rerender one assistant turn through the production
307 +/// renderer/output-processor boundary.
308 +async fn run_roundtrip(
309 + case: &RoundtripCase,
310 + backends: &vllm_chat::LoadedModelBackends,
311 + request: &ChatRequest,
312 + assistant: AssistantMessage,
313 +) -> Result<RoundtripResult> {
314 + let renderer = backends.chat_backend.chat_renderer();
315 + let (prompt, closed_completion_text) =
316 + render_closed_completion(renderer.as_ref(), request, &assistant)?;
317 + let completion_body = closed_completion_text
318 + .strip_suffix(case.assistant_stop_suffix)
319 + .with_context(|| {
320 + format!(
321 + "closed assistant completion did not end with {:?}: {:?}",
322 + case.assistant_stop_suffix, closed_completion_text
323 + )
324 + })?;
325 +
326 + let parsed_message =
327 + parse_completion(case, backends, request, &prompt, completion_body).await?;
328 + let (_, rerendered_closed_completion) =
329 + render_closed_completion(renderer.as_ref(), request, &parsed_message)?;
330 +
331 + Ok(RoundtripResult {
332 + parsed_message,
333 + closed_completion: closed_completion_text,
334 + rerendered_closed_completion,
335 + })
336 +}
337 +
338 +/// Render `history` as a production prompt and `history + assistant` as closed
339 +/// history, then return the production prompt and assistant-completion suffix.
340 +fn render_closed_completion(
341 + renderer: &dyn vllm_chat::ChatRenderer,
342 + base_request: &ChatRequest,
343 + assistant: &AssistantMessage,
344 +) -> Result<(String, String)> {
345 + let mut prompt_request = base_request.clone();
346 + prompt_request.chat_options.generation_prompt_mode = GenerationPromptMode::StartNewAssistant;
347 + let prompt = render_text(renderer, &prompt_request).context("failed to render prompt")?;
348 +
349 + let mut full_request = base_request.clone();
350 + full_request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
351 + full_request.messages.push(ChatMessage::from(assistant.clone()));
352 + let full = render_text(renderer, &full_request).context("failed to render full prompt")?;
353 +
354 + ensure!(
355 + full.starts_with(&prompt),
356 + "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}"
357 + );
358 + let completion = full[prompt.len()..].to_string();
359 +
360 + Ok((prompt, completion))
361 +}
362 +
363 +/// Render one chat request and require a text prompt.
364 +fn render_text(renderer: &dyn vllm_chat::ChatRenderer, request: &ChatRequest) -> Result<String> {
365 + match renderer.render(request)?.prompt {
366 + Prompt::Text(text) => Ok(text),
367 + other => bail!("roundtrip tests expect text prompts, got {other:?}"),
368 + }
369 +}
370 +
371 +/// Feed one rendered assistant completion body into the real output processor
372 +/// and collect its terminal assistant message.
373 +async fn parse_completion(
374 + case: &RoundtripCase,
375 + backends: &vllm_chat::LoadedModelBackends,
376 + base_request: &ChatRequest,
377 + prompt: &str,
378 + completion_body: &str,
379 +) -> Result<AssistantMessage> {
380 + let tokenizer = backends.text_backend.tokenizer();
381 + let prompt_token_ids = tokenizer
382 + .encode(prompt, base_request.add_special_tokens)
383 + .context("failed to encode rendered prompt")?;
384 +
385 + let mut request = base_request.clone();
386 + let processor = backends.chat_backend.new_chat_output_processor(
387 + &mut request,
388 + NewChatOutputProcessorOptions {
389 + tool_call_parser: &case.tool_call_parser,
390 + reasoning_parser: &case.reasoning_parser,
391 + },
392 + )?;
393 +
394 + let decoded = decoded_completion_stream(prompt_token_ids, completion_body);
395 + let mut events = processor.process(decoded)?;
396 +
397 + while let Some(event) = events.next().await {
398 + if let ChatEvent::Done { message, .. } = event? {
399 + // TODO: currently our parsers are not very strict about preserving or trimming
400 + // whitespace, so we trim here to avoid roundtrip failures due to
401 + // insignificant whitespace differences. However, this may hurt token-level
402 + // fidelity so we should consider improving them.
403 + return Ok(message.trim());
404 + }
405 + }
406 +
407 + bail!("output processor finished without a Done event")
408 +}
409 +
410 +/// Build a decoded-text stream from an already-rendered completion body.
411 +///
412 +/// The first event carries real prompt token ids so reasoning parsers can
413 +/// initialize from the same prompt boundary production uses. Completion text is
414 +/// split into small chunks to exercise streaming parser state across marker
415 +/// and JSON boundaries.
416 +fn decoded_completion_stream(
417 + prompt_token_ids: Vec<u32>,
418 + completion_body: &str,
419 +) -> Pin<Box<dyn Stream<Item = vllm_chat::Result<DecodedTextEvent>> + Send>> {
420 + let prompt_token_count = prompt_token_ids.len();
421 + let mut events = vec![DecodedTextEvent::Start {
422 + prompt_token_ids: Arc::from(prompt_token_ids.into_boxed_slice()),
423 + prompt_logprobs: None,
424 + }];
425 +
426 + let chunks = split_by_chars(completion_body, 7);
427 + if chunks.is_empty() {
428 + events.push({
429 + DecodedTextEvent::TextDelta {
430 + delta: String::new(),
431 + token_ids: Vec::new(),
432 + logprobs: None,
433 + finished: Some(Finished {
434 + prompt_token_count: 0,
435 + output_token_count: 0,
436 + finish_reason: FinishReason::stop_eos(),
437 + kv_transfer_params: None,
438 + }),
439 + }
440 + });
441 + } else {
442 + let last_index = chunks.len() - 1;
443 + for (index, chunk) in chunks.into_iter().enumerate() {
444 + let finished = (index == last_index).then(|| Finished {
445 + prompt_token_count,
446 + output_token_count: completion_body.chars().count(),
447 + finish_reason: FinishReason::stop_eos(),
448 + kv_transfer_params: None,
449 + });
450 + events.push(DecodedTextEvent::TextDelta {
451 + delta: chunk,
452 + token_ids: Vec::new(),
453 + logprobs: None,
454 + finished,
455 + });
456 + }
457 + }
458 +
459 + stream::iter(events).map(Ok).boxed()
460 +}
461 +
462 +/// Split text into chunks containing at most `chunk_chars` Unicode scalar
463 +/// values.
464 +fn split_by_chars(text: &str, chunk_chars: usize) -> Vec<String> {
465 + let mut chunks = Vec::new();
466 + let mut start = 0;
467 + let mut count = 0;
468 +
469 + for (index, _) in text.char_indices() {
470 + if count == chunk_chars {
471 + chunks.push(text[start..index].to_string());
472 + start = index;
473 + count = 0;
474 + }
475 + count += 1;
476 + }
477 +
478 + if start < text.len() {
479 + chunks.push(text[start..].to_string());
480 + }
481 +
482 + chunks
483 +}
484 +
485 +/// Build a chat request fixture with parser-enabling tool-choice semantics.
486 +fn roundtrip_request(
487 + request_id: impl Into<String>,
488 + messages: Vec<ChatMessage>,
489 + tools: Vec<ChatTool>,
490 +) -> ChatRequest {
491 + let mut request = ChatRequest {
492 + request_id: request_id.into(),
493 + messages,
494 + tool_choice: if tools.is_empty() {
495 + ChatToolChoice::None
496 + } else {
497 + ChatToolChoice::Auto
498 + },
499 + tools,
500 + ..ChatRequest::for_test()
501 + };
502 +
503 + // Enable thinking for some models so that rendering and parsing the reasoning block is
504 + // exercised in the roundtrip.
505 + for key in ["thinking", "enable_thinking"] {
506 + request.chat_options.template_kwargs.insert(key.to_string(), true.into());
507 + }
508 +
509 + request
510 +}
511 +
512 +/// Return the function tools used by the multiple-tool-call fixture.
513 +fn test_tools() -> Vec<ChatTool> {
514 + vec![
515 + ChatTool {
516 + name: "get_weather".to_string(),
517 + description: Some("Get weather for a location".to_string()),
518 + parameters: serde_json::json!({
519 + "type": "object",
520 + "properties": {
521 + "location": { "type": "string" }
522 + },
523 + "required": ["location"]
524 + }),
525 + strict: None,
526 + },
527 + ChatTool {
528 + name: "add".to_string(),
529 + description: Some("Add two integers".to_string()),
530 + parameters: serde_json::json!({
531 + "type": "object",
532 + "properties": {
533 + "y": { "type": "number" },
534 + "x": { "type": "number" }
535 + },
536 + "required": ["y", "x"]
537 + }),
538 + strict: None,
539 + },
540 + ]
541 +}
rust/src/text/Cargo.toml
+1
@@ -26,6 +26,7 @@ vllm-tokenizer.workspace = true
26 [dev-dependencies]
27 expect-test.workspace = true
28 futures.workspace = true
29 +serial_test.workspace = true
30 tempfile.workspace = true
31 tokio.workspace = true
32 vllm-llm = { workspace = true, features = ["test-util"] }
rust/src/text/src/backend/hf/model_files.rs
+1 -1
@@ -401,7 +401,7 @@ mod tests {
401 }
402
403 #[tokio::test]
404 - #[ignore = "requires network access to Hugging Face and downloads the real Kimi K2.5 tokenizer"]
404 + #[ignore = "too slow for CI and requires network access to Hugging Face"]
405 async fn tiktoken_real_kimi_k25_tokenizer_files_load_and_handle_special_tokens() {
406 let files = ResolvedModelFiles::new("moonshotai/Kimi-K2.5")
407 .await
rust/src/text/src/lower.rs
+14 -9
@@ -235,6 +235,8 @@ fn merge_unique_token_ids(
235 mod tests {
236 use std::collections::BTreeSet;
237
238 + use serial_test::file_serial;
239 +
240 use super::*;
241 use crate::backend::hf::HfTextBackend;
242 use crate::backend::{SamplingHints, TextBackend as _};
@@ -386,7 +388,7 @@ mod tests {
388 }
389
390 #[tokio::test]
389 - #[ignore = "requires network access to Hugging Face"]
391 + #[file_serial(hf_qwen3)]
392 async fn lower_text_request_uses_real_qwen_generation_defaults() {
393 let backend = HfTextBackend::from_model("Qwen/Qwen3-0.6B")
394 .await
@@ -410,12 +412,8 @@ mod tests {
412 default_top_k: Some(
413 20,
414 ),
413 - default_min_p: Some(
414 - 0.1,
415 - ),
416 - default_repetition_penalty: Some(
417 - 1.2,
418 - ),
415 + default_min_p: None,
416 + default_repetition_penalty: None,
417 default_max_tokens: None,
418 max_model_len: Some(
419 40960,
@@ -439,10 +437,10 @@ mod tests {
437 min_tokens: 0,
438 logprobs: None,
439 prompt_logprobs: None,
442 - min_p: 0.1,
440 + min_p: 0.0,
441 frequency_penalty: 0.0,
442 presence_penalty: 0.0,
445 - repetition_penalty: 1.2,
443 + repetition_penalty: 1.0,
444 stop_token_ids: [
445 151643,
446 ],
@@ -453,6 +451,13 @@ mod tests {
451 151643,
452 151645,
453 },
454 + logit_bias: None,
455 + allowed_token_ids: None,
456 + bad_words_token_ids: None,
457 + structured_outputs: None,
458 + logprob_token_ids: None,
459 + skip_reading_prefix_cache: None,
460 + extra_args: None,
461 }
462 "#]]
463 .assert_debug_eq(&params);
rust/src/tool-parser/src/minimax_m2.rs
+48 -6
@@ -1,5 +1,5 @@
1 use winnow::ascii::{multispace0 as ws0, multispace1 as ws1};
2 -use winnow::combinator::{alt, delimited, repeat, seq, terminated};
2 +use winnow::combinator::{alt, delimited, eof, repeat, seq, terminated};
3 use winnow::prelude::*;
4 use winnow::stream::Partial;
5 use winnow::token::{literal, rest, take_until};
@@ -183,16 +183,17 @@ fn tool_block_end_event(input: &mut MinimaxM2Input<'_>) -> ModalResult<MinimaxM2
183
184 /// Parse a complete MiniMax M2 invoke block.
185 fn invoke_event(input: &mut MinimaxM2Input<'_>) -> ModalResult<MinimaxM2Event> {
186 - let (name, raw_params) = seq!(
186 + let (name, body) = seq!(
187 _: ws0,
188 _: literal(INVOKE_START),
189 _: (ws1, literal("name=")),
190 - attr_value,
190 + partial_attr_value,
191 _: literal(">"),
192 - repeat(0.., terminated(parameter, ws0)),
192 + take_until(0.., INVOKE_END),
193 _: literal(INVOKE_END),
194 )
195 .parse_next(input)?;
196 + let raw_params = parse_invoke_params(body)?;
197
198 Ok(MinimaxM2Event::Invoke {
199 name: name.trim().to_string(),
@@ -200,8 +201,14 @@ fn invoke_event(input: &mut MinimaxM2Input<'_>) -> ModalResult<MinimaxM2Event> {
201 })
202 }
203
204 +/// Parse all parameter blocks inside a complete MiniMax M2 invoke body.
205 +fn parse_invoke_params(invoke_body: &str) -> ModalResult<Vec<(String, String)>> {
206 + let mut input = invoke_body;
207 + delimited(ws0, repeat(0.., terminated(parameter, ws0)), eof).parse_next(&mut input)
208 +}
209 +
210 /// Parse a MiniMax M2 parameter block.
204 -fn parameter(input: &mut MinimaxM2Input<'_>) -> ModalResult<(String, String)> {
211 +fn parameter(input: &mut &str) -> ModalResult<(String, String)> {
212 let (name, value) = seq!(
213 _: literal(PARAMETER_START),
214 _: (ws1, literal("name=")),
@@ -216,7 +223,17 @@ fn parameter(input: &mut MinimaxM2Input<'_>) -> ModalResult<(String, String)> {
223 }
224
225 /// Parse a quoted or unquoted XML attribute value.
219 -fn attr_value<'i>(input: &mut MinimaxM2Input<'i>) -> ModalResult<&'i str> {
226 +fn attr_value<'i>(input: &mut &'i str) -> ModalResult<&'i str> {
227 + alt((
228 + delimited(literal("\""), take_until(1.., "\""), literal("\"")),
229 + delimited(literal("'"), take_until(1.., "'"), literal("'")),
230 + take_until(1.., ">"),
231 + ))
232 + .parse_next(input)
233 +}
234 +
235 +/// Parse a quoted or unquoted XML attribute value from partial streaming input.
236 +fn partial_attr_value<'i>(input: &mut MinimaxM2Input<'i>) -> ModalResult<&'i str> {
237 alt((
238 delimited(literal("\""), take_until(1.., "\""), literal("\"")),
239 delimited(literal("'"), take_until(1.., "'"), literal("'")),
@@ -470,6 +487,31 @@ mod tests {
487 assert_eq!(result.calls[1].tool_index, 1);
488 }
489
490 + #[test]
491 + fn minimax_m2_streaming_handles_template_whitespace_and_split_parameters() {
492 + let text = concat!(
493 + "I will call the tools.\n",
494 + "<minimax:tool_call>\n",
495 + "<invoke name=\"get_weather\">\n",
496 + "<parameter name=\"city\">Seattle</parameter>\n",
497 + "</invoke>\n",
498 + "<invoke name=\"get_weather\">\n",
499 + "<parameter name=\"city\">NYC</parameter>\n",
500 + "</invoke>\n",
501 + "</minimax:tool_call>",
502 + );
503 + let chunks = split_by_chars(text, 7);
504 + let mut parser = MinimaxM2ToolParser::new(&test_tools());
505 + let result = collect_stream(&mut parser, &chunks);
506 +
507 + assert_eq!(result.normal_text, "I will call the tools.\n");
508 + assert_eq!(result.calls.len(), 2);
509 + assert_eq!(result.calls[0].tool_index, 0);
510 + assert_eq!(result.calls[0].name.as_deref(), Some("get_weather"));
511 + assert_eq!(result.calls[1].tool_index, 1);
512 + assert_eq!(result.calls[1].name.as_deref(), Some("get_weather"));
513 + }
514 +
515 #[test]
516 fn minimax_m2_streaming_ignores_text_after_tool_block() {
517 let text = format!(
rust/src/tool-parser/src/parameters.rs
+19 -12
@@ -224,10 +224,11 @@ fn convert_value(param_type: &JsonParamType, value: &str) -> Option<Value> {
224
225 /// Convert one raw string value to a JSON number.
226 fn convert_number(value: &str) -> Option<Value> {
227 - if let Ok(parsed) = value.parse::<i64>() {
228 - return Some(Value::Number(Number::from(parsed)));
229 - }
230 - Number::from_f64(value.parse::<f64>().ok()?).map(Value::Number)
227 + serde_json::from_str::<Number>(value)
228 + .or_else(|_| value.parse::<i64>().map(Number::from))
229 + .or_else(|_| value.parse::<f64>().ok().and_then(Number::from_f64).ok_or(()))
230 + .ok()
231 + .map(Value::Number)
232 }
233
234 /// Convert one raw string value to a boolean.
@@ -304,7 +305,7 @@ mod tests {
305 }
306
307 #[test]
307 - fn number_conversion_parses_int_then_float() {
308 + fn number_conversion_preserves_json_number_spelling_with_legacy_fallback() {
309 let params = ToolSchema::from_schema(&json!({
310 "type": "object",
311 "properties": {
@@ -312,17 +313,23 @@ mod tests {
313 }
314 }));
315
315 - assert_eq!(params.convert("value", "5"), json!(5));
316 - assert_eq!(params.convert("value", "5.0"), json!(5.0));
317 - assert_eq!(params.convert("value", "5."), json!(5.0));
318 - assert_eq!(params.convert("value", "+1"), json!(1));
319 - assert_eq!(params.convert("value", "+1.0"), json!(1.0));
316 + assert_eq!(converted_number_text(&params, "5"), "5");
317 + assert_eq!(converted_number_text(&params, "5.0"), "5.0");
318 + assert_eq!(converted_number_text(&params, "5.00"), "5.00");
319 + assert_eq!(converted_number_text(&params, "1e0"), "1e+0");
320 + assert_eq!(converted_number_text(&params, "5."), "5.0");
321 + assert_eq!(converted_number_text(&params, "+1"), "1");
322 + assert_eq!(converted_number_text(&params, "+1.0"), "1.0");
323 assert_eq!(
321 - params.convert("value", "9223372036854775807.5"),
322 - json!(9223372036854775808.0)
324 + converted_number_text(&params, "9223372036854775807.5"),
325 + "9223372036854775807.5"
326 );
327 }
328
329 + fn converted_number_text(params: &ToolSchema, value: &str) -> String {
330 + serde_json::to_string(&params.convert("value", value)).unwrap()
331 + }
332 +
333 #[test]
334 fn converts_upstream_aliases() {
335 let params = ToolSchema::from_schema(&json!({