A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 02:15 UTC
[Rust Frontend] Fix several hf chat template rendering issues (#44311)
Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Bugen Zhao committed
Jun 3, 2026 at 16:04 UTC
449be4f93484d2b8b9ae06bb2f86c6e42c6760d7
9 files changed
+183
-56
rust/Cargo.lock
+1
@@ -5622,6 +5622,7 @@ dependencies = [
5622
"expect-test",
5623
"futures",
5624
"half",
5625
+ "indexmap 2.13.0",
5626
"itertools 0.14.0",
5627
"llm-multimodal",
5628
"minijinja",
rust/Cargo.toml
+2
-1
@@ -43,6 +43,7 @@ half = { version = "2.7.1", features = ["bytemuck"] }
43
hex = "0.4.3"
44
hf-hub = { version = "0.5.0", features = ["tokio"] }
45
http-body = "1.0.1"
46
+indexmap = "2.13.0"
47
itertools = "0.14.0"
48
libc = "0.2.177"
49
llm-multimodal = { git = "https://github.com/vllm-project/llm-multimodal", rev = "5b558989844d1c7af3e43d0f604069ffd9c06320" }
@@ -69,7 +70,7 @@ rustc-hash = "1.1.0"
70
serde = { version = "1.0.228", features = ["derive"] }
71
serde-json-fmt = "0.1.0"
72
serde_default = "0.2.0"
72
-serde_json = { version = "1.0.145", features = ["arbitrary_precision", "preserve_order"] }
73
+serde_json = { version = "1.0.145", features = ["preserve_order"] }
74
serde_repr = "0.1.20"
75
serde_tuple = "1.1.3"
76
serde_with = "3.18.0"
rust/src/chat/Cargo.toml
+1
@@ -10,6 +10,7 @@ asynk-strim-attr.workspace = true
10
easy-ext.workspace = true
11
futures.workspace = true
12
half.workspace = true
13
+indexmap.workspace = true
14
itertools.workspace = true
15
llm-multimodal.workspace = true
16
minijinja.workspace = true
rust/src/chat/src/renderer/hf/mod.rs
+33
-7
@@ -1,7 +1,7 @@
1
use std::collections::HashMap;
2
3
use serde::Serialize;
4
-use serde_json::Value;
4
+use serde_json::Value as JsonValue;
5
use thiserror_ext::AsReport as _;
6
use tracing::{info, trace, warn};
7
use vllm_text::Prompt;
@@ -13,6 +13,7 @@ use self::format::{
13
ChatTemplateContentFormat, ChatTemplateContentFormatOption as ContentFormatOption,
14
};
15
use self::template::{CompiledChatTemplate, TemplateContext};
16
+use self::value::{TemplateValue, to_template_value};
17
use super::{ChatRenderer, RenderedPrompt};
18
use crate::error::Result;
19
use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
@@ -24,6 +25,7 @@ mod error;
25
mod format;
26
mod template;
27
mod tojson;
28
+mod value;
29
30
pub use template::{load_chat_template, resolve_chat_template};
31
@@ -38,7 +40,7 @@ pub struct MultimodalRenderInfo {
40
/// state.
41
pub struct HfChatRenderer {
42
default_template: Option<CompiledChatTemplate>,
41
- default_template_kwargs: HashMap<String, Value>,
43
+ default_template_kwargs: HashMap<String, JsonValue>,
44
content_format: ContentFormatOption,
45
special_tokens: Option<HfSpecialTokens>,
46
multimodal: Option<MultimodalRenderInfo>,
@@ -48,7 +50,7 @@ impl HfChatRenderer {
50
/// Create a renderer from the given template string.
51
pub fn new(
52
template: Option<String>,
51
- default_template_kwargs: HashMap<String, Value>,
53
+ default_template_kwargs: HashMap<String, JsonValue>,
54
content_format: ContentFormatOption,
55
) -> Result<Self> {
56
Ok(Self {
@@ -245,7 +247,7 @@ struct TemplateToolCall {
247
#[derive(Debug, Serialize)]
248
struct TemplateToolFunction {
249
name: String,
248
- arguments: Value,
250
+ arguments: TemplateValue,
251
}
252
253
#[derive(Debug, Serialize)]
@@ -259,7 +261,7 @@ pub(super) struct TemplateTool {
261
struct TemplateToolDefinition {
262
name: String,
263
description: Option<String>,
262
- parameters: Value,
264
+ parameters: TemplateValue,
265
strict: Option<bool>,
266
}
267
@@ -345,13 +347,14 @@ fn to_template_tool_calls(
347
let mut tool_calls = Vec::new();
348
349
for tool_call in content.tool_calls() {
348
- let arguments = serde_json::from_str::<Value>(&tool_call.arguments).map_err(|error| {
350
+ let arguments = serde_json::from_str(&tool_call.arguments).map_err(|error| {
351
Error::ChatTemplate(format!(
352
"assistant tool call `{}` has invalid JSON arguments: {}",
353
tool_call.id,
354
error.as_report()
355
))
356
})?;
357
+ let arguments = to_template_value(arguments);
358
359
tool_calls.push(TemplateToolCall {
360
id: tool_call.id.clone(),
@@ -434,7 +437,7 @@ fn to_template_tools(tools: &[ChatTool]) -> Vec<TemplateTool> {
437
function: TemplateToolDefinition {
438
name: tool.name.clone(),
439
description: tool.description.clone(),
437
- parameters: tool.parameters.clone(),
440
+ parameters: to_template_value(tool.parameters.clone()),
441
strict: tool.strict,
442
},
443
})
@@ -909,6 +912,29 @@ mod tests {
912
assert_eq!(rendered, "get_weather|Paris|call_1|Sunny");
913
}
914
915
+ #[test]
916
+ fn chat_template_tool_call_argument_items_method_is_not_shadowed_by_field() {
917
+ let request = sample_request(vec![ChatMessage::assistant_blocks(vec![
918
+ AssistantContentBlock::ToolCall(crate::AssistantToolCall {
919
+ id: "call_1".to_string(),
920
+ name: "add".to_string(),
921
+ arguments: r#"{"items":"operands","x":2,"y":1.0}"#.to_string(),
922
+ }),
923
+ ])]);
924
+
925
+ let rendered = render(
926
+ Some(
927
+ "{%- set arguments = messages[0].tool_calls[0].function.arguments -%}
928
+{%- for key, value in arguments.items() -%}{{ key }}={{ value }};{%- endfor -%}
929
+|{{ arguments['items'] }}",
930
+ ),
931
+ &request,
932
+ )
933
+ .unwrap();
934
+
935
+ assert_eq!(rendered, "items=operands;x=2;y=1.0;|operands");
936
+ }
937
+
938
#[test]
939
fn qwen35_template_renders_prefilled_reasoning_start_when_thinking_enabled() {
940
let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
rust/src/chat/src/renderer/hf/tojson.rs
+18
-2
@@ -208,11 +208,27 @@ mod tests {
208
}
209
210
#[test]
211
- fn tojson_preserves_arbitrary_precision_number_spelling() {
211
+ fn tojson_uses_standard_serde_json_number_spelling() {
212
let payload = serde_json::from_str(r#"{"x":2,"y":1.00}"#).unwrap();
213
let rendered = render("{{ payload|tojson }}", payload);
214
215
- assert_eq!(rendered, "{\"x\": 2, \"y\": 1.00}");
215
+ // TODO: we cannot preserve the original number precision by enabling `serde_json`'s
216
+ // `arbitrary_precision` feature, otherwise the following test
217
+ // `serialized_json_numbers_do_not_leak_serde_private_representation` will fail.
218
+ // See issue: https://github.com/mitsuhiko/minijinja/issues/641
219
+ assert_eq!(rendered, "{\"x\": 2, \"y\": 1.0}");
220
+ }
221
+
222
+ #[test]
223
+ fn serialized_json_numbers_do_not_leak_serde_private_representation() {
224
+ let payload: serde_json::Value = serde_json::from_str(r#"{"x":2,"y":1.00}"#).unwrap();
225
+ let rendered = render("{{ payload }}", payload);
226
+
227
+ // TODO: we cannot preserve the original number precision by enabling `serde_json`'s
228
+ // `arbitrary_precision` feature, otherwise this will fail.
229
+ // See issue: https://github.com/mitsuhiko/minijinja/issues/641
230
+ assert!(!rendered.contains("$serde_json::private::Number"));
231
+ assert_eq!(rendered, r#"{"x": 2, "y": 1.0}"#);
232
}
233
234
#[test]
rust/src/chat/src/renderer/hf/value.rs
new
+77
@@ -0,0 +1,77 @@
1
+use std::sync::Arc;
2
+
3
+use indexmap::IndexMap;
4
+use minijinja::value::{Enumerator, Object, ObjectExt, ObjectRepr};
5
+use minijinja::{Error as TemplateError, ErrorKind as TemplateErrorKind, State};
6
+use serde::Serialize;
7
+use serde_json::Value as JsonValue;
8
+
9
+/// A wrapper around `minijinja::Value` that can be constructed with `to_template_value` and used
10
+/// as a value in the chat template.
11
+#[derive(Debug, Serialize)]
12
+#[serde(transparent)]
13
+pub(super) struct TemplateValue(minijinja::Value);
14
+
15
+pub(super) fn to_template_value(value: JsonValue) -> TemplateValue {
16
+ TemplateValue(match value {
17
+ JsonValue::Array(values) => values
18
+ .into_iter()
19
+ .map(to_template_value)
20
+ .map(|value| value.0)
21
+ .collect::<minijinja::Value>(),
22
+ JsonValue::Object(values) => minijinja::Value::from_object(TemplateMap(
23
+ values
24
+ .into_iter()
25
+ .map(|(key, value)| (key, to_template_value(value).0))
26
+ .collect(),
27
+ )),
28
+ // For primitive values, directly convert them to `minijinja::Value` using `from_serialize`.
29
+ value => minijinja::Value::from_serialize(value),
30
+ })
31
+}
32
+
33
+/// A custom map type that always returns `UnknownMethod` for method calls, so that pycompat can
34
+/// always handle dict methods through the unknown-method callback.
35
+///
36
+/// Use `IndexMap` to preserve the original key order when iterating.
37
+///
38
+/// MiniJinja's default map can resolve a same-named field before Python dict methods. HF templates
39
+/// commonly call `dict.items()`, which would fail if the map had an `items` field.
40
+/// See issue: https://github.com/mitsuhiko/minijinja/issues/903
41
+#[derive(Debug)]
42
+struct TemplateMap(IndexMap<String, minijinja::Value>);
43
+
44
+impl Object for TemplateMap {
45
+ fn repr(self: &Arc<Self>) -> ObjectRepr {
46
+ ObjectRepr::Map
47
+ }
48
+
49
+ fn get_value(self: &Arc<Self>, key: &minijinja::Value) -> Option<minijinja::Value> {
50
+ self.0.get(key.as_str()?).cloned()
51
+ }
52
+
53
+ fn get_value_by_str(self: &Arc<Self>, key: &str) -> Option<minijinja::Value> {
54
+ self.0.get(key).cloned()
55
+ }
56
+
57
+ fn enumerate(self: &Arc<Self>) -> Enumerator {
58
+ self.mapped_rev_enumerator(|this| {
59
+ Box::new(this.0.keys().map(|key| minijinja::Value::from(key.as_str())))
60
+ })
61
+ }
62
+
63
+ fn enumerator_len(self: &Arc<Self>) -> Option<usize> {
64
+ Some(self.0.len())
65
+ }
66
+
67
+ fn call_method(
68
+ self: &Arc<Self>,
69
+ _state: &State<'_, '_>,
70
+ _method: &str,
71
+ _args: &[minijinja::Value],
72
+ ) -> std::result::Result<minijinja::Value, TemplateError> {
73
+ // Always return `UnknownMethod` for method calls,
74
+ // so that pycompat can handle dict methods through the unknown-method callback.
75
+ Err(TemplateError::from(TemplateErrorKind::UnknownMethod))
76
+ }
77
+}
rust/src/chat/tests/roundtrip.rs
+12
-7
@@ -233,7 +233,7 @@ async fn run_roundtrip_tool_call_mix(case: RoundtripCase) -> Result<()> {
233
"roundtrip-reasoning-tools",
234
vec![ChatMessage::text(
235
ChatRole::User,
236
- "Check Shanghai weather and add 1.00 plus 2.",
236
+ "Check Shanghai weather and add 1.0 plus 2.",
237
)],
238
test_tools(),
239
Some(true), // always enable thinking in this fixture
@@ -261,9 +261,10 @@ async fn run_roundtrip_tool_call_mix(case: RoundtripCase) -> Result<()> {
261
AssistantContentBlock::ToolCall(AssistantToolCall {
262
id: "functions.add:1".to_string(),
263
name: "add".to_string(),
264
- // Intentionally use a non-lexical order of keys and a different number
265
- // formatting style to verify text-level fidelity of the roundtrip.
266
- arguments: r#"{"y":1.00,"x":2}"#.to_string(),
264
+ // Intentionally use a non-lexical order of keys to verify text-level
265
+ // fidelity of the roundtrip where JSON formatting remains stable. The
266
+ // `items` key also exercises templates that call `arguments.items()`.
267
+ arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(),
268
}),
269
],
270
},
@@ -291,7 +292,7 @@ async fn run_roundtrip_tool_call_mix(case: RoundtripCase) -> Result<()> {
292
assert_eq!(tool_calls[1].name, "add");
293
assert_eq!(
294
tool_calls[1].arguments,
294
- expected_arguments(&case, r#"{"y": 1.00, "x": 2}"#)?,
295
+ expected_arguments(&case, r#"{"y": 1.0, "x": 2, "items": ["left", "right"]}"#)?,
296
);
297
298
assert_eq!(
@@ -585,9 +586,13 @@ fn test_tools() -> Vec<ChatTool> {
586
"type": "object",
587
"properties": {
588
"y": { "type": "number" },
588
- "x": { "type": "number" }
589
+ "x": { "type": "number" },
590
+ "items": {
591
+ "type": "array",
592
+ "items": { "type": "string" }
593
+ }
594
},
590
- "required": ["y", "x"]
595
+ "required": ["y", "x", "items"]
596
}),
597
strict: None,
598
},
rust/src/tool-parser/src/json/internlm2.rs
+27
-33
@@ -34,26 +34,23 @@ const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
34
/// This Rust port intentionally diverges from
35
/// `vllm/tool_parsers/internlm2_tool_parser.py` in two user-visible ways:
36
///
37
-/// - **Parallel tool calls are supported.** Python silently drops every
38
-/// `<|action_start|>` block after the first (`current_tool_id > 0` returns
39
-/// an empty delta); this parser emits every well-formed block with
40
-/// incrementing `tool_index`. Models that legitimately emit multiple action
41
-/// blocks therefore produce more tool calls under Rust than under Python.
42
-/// - **End-marker bytes inside JSON string values are preserved.** Python
43
-/// does `action.split("<|action_end|>")[0]` which truncates regardless of
44
-/// JSON context; this parser scans matched braces and quotes so a literal
45
-/// `<|action_end|>` inside an arguments string is forwarded intact.
37
+/// - **Parallel tool calls are supported.** Python silently drops every `<|action_start|>` block
38
+/// after the first (`current_tool_id > 0` returns an empty delta); this parser emits every
39
+/// well-formed block with incrementing `tool_index`. Models that legitimately emit multiple
40
+/// action blocks therefore produce more tool calls under Rust than under Python.
41
+/// - **End-marker bytes inside JSON string values are preserved.** Python does
42
+/// `action.split("<|action_end|>")[0]` which truncates regardless of JSON context; this parser
43
+/// scans matched braces and quotes so a literal `<|action_end|>` inside an arguments string is
44
+/// forwarded intact.
45
/// - **Only whitespace is allowed before the `{`.** Python's non-streaming
47
-/// `action[action.find("{"):]` drops any bytes before the first `{`, but
48
-/// its streaming path has no equivalent and the model format always emits
49
-/// `<|plugin|>{...`; this parser allows only whitespace there, matching the
50
-/// other JSON parsers in this crate.
51
-/// - **Truncated tool calls error rather than silently dropping.** Python's
52
-/// streaming wrapper swallows mid-stream errors with `except Exception:
53
-/// return None` (logging a traceback) while its non-streaming path raises
54
-/// `JSONDecodeError`; this parser returns an `incomplete InternLM2 tool
55
-/// call` error from `finish()`, matching the other JSON parsers and Python's
56
-/// non-streaming behavior.
46
+/// `action[action.find("{"):]` drops any bytes before the first `{`, but its streaming path has
47
+/// no equivalent and the model format always emits `<|plugin|>{...`; this parser allows only
48
+/// whitespace there, matching the other JSON parsers in this crate.
49
+/// - **Truncated tool calls error rather than silently dropping.** Python's streaming wrapper
50
+/// swallows mid-stream errors with `except Exception: return None` (logging a traceback) while
51
+/// its non-streaming path raises `JSONDecodeError`; this parser returns an `incomplete InternLM2
52
+/// tool call` error from `finish()`, matching the other JSON parsers and Python's non-streaming
53
+/// behavior.
54
///
55
/// # Known unaddressed divergences (TODO)
56
///
@@ -63,21 +60,18 @@ const INTERNLM2_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
60
/// Qwen as well. If a real-world InternLM2 deployment hits one of these,
61
/// prioritize the corresponding fix.
62
///
66
-/// - **Arguments value type.** The shared core requires the arguments value
67
-/// to be a JSON object (`take_json_object` rejects anything not starting
68
-/// with `{`). Python's `json.dumps(action_dict.get("parameters", ...))`
69
-/// accepts `null`, arrays, strings, and numbers and round-trips them
70
-/// verbatim. Models that legitimately emit `"parameters":null` will hard-
63
+/// - **Arguments value type.** The shared core requires the arguments value to be a JSON object
64
+/// (`take_json_object` rejects anything not starting with `{`). Python's
65
+/// `json.dumps(action_dict.get("parameters", ...))` accepts `null`, arrays, strings, and numbers
66
+/// and round-trips them verbatim. Models that legitimately emit `"parameters":null` will hard-
67
/// fail under Rust.
72
-/// - **Unknown arguments key.** Python falls back to `{}` via
73
-/// `action_dict.get("parameters", action_dict.get("arguments", {}))` when
74
-/// neither key is present; the Rust header parser raises
75
-/// `parsing failed: invalid InternLM2` for any unrecognized key. A model
76
-/// that emits a typo (e.g. `"params"`) breaks the whole response.
77
-/// - **Field order independence.** The header parser requires the JSON keys
78
-/// to appear in the order `name` then arguments key. Python's
79
-/// `json.loads` + `dict.get` is order-independent, so a model emitting
80
-/// `{"parameters":{...},"name":"foo"}` parses in Python but fails in Rust.
68
+/// - **Unknown arguments key.** Python falls back to `{}` via `action_dict.get("parameters",
69
+/// action_dict.get("arguments", {}))` when neither key is present; the Rust header parser raises
70
+/// `parsing failed: invalid InternLM2` for any unrecognized key. A model that emits a typo (e.g.
71
+/// `"params"`) breaks the whole response.
72
+/// - **Field order independence.** The header parser requires the JSON keys to appear in the order
73
+/// `name` then arguments key. Python's `json.loads` + `dict.get` is order-independent, so a model
74
+/// emitting `{"parameters":{...},"name":"foo"}` parses in Python but fails in Rust.
75
pub struct Internlm2ToolParser {
76
inner: JsonToolCallParser,
77
}
rust/src/tool-parser/src/parameters.rs
+12
-6
@@ -501,15 +501,21 @@ mod tests {
501
502
assert_eq!(converted_number_text(¶ms, "5"), "5");
503
assert_eq!(converted_number_text(¶ms, "5.0"), "5.0");
504
- assert_eq!(converted_number_text(¶ms, "5.00"), "5.00");
505
- assert_eq!(converted_number_text(¶ms, "1e0"), "1e+0");
504
assert_eq!(converted_number_text(¶ms, "5."), "5.0");
505
assert_eq!(converted_number_text(¶ms, "+1"), "1");
506
assert_eq!(converted_number_text(¶ms, "+1.0"), "1.0");
509
- assert_eq!(
510
- converted_number_text(¶ms, "9223372036854775807.5"),
511
- "9223372036854775807.5"
512
- );
507
+
508
+ // TODO: we cannot preserve the original number precision by enabling `serde_json`'s
509
+ // `arbitrary_precision` feature, otherwise the test
510
+ // `serialized_json_numbers_do_not_leak_serde_private_representation` will fail.
511
+ // See issue: https://github.com/mitsuhiko/minijinja/issues/641
512
+
513
+ // assert_eq!(converted_number_text(¶ms, "5.00"), "5.00");
514
+ // assert_eq!(converted_number_text(¶ms, "1e0"), "1e+0");
515
+ // assert_eq!(
516
+ // converted_number_text(¶ms, "9223372036854775807.5"),
517
+ // "9223372036854775807.5"
518
+ // );
519
}
520
521
fn converted_number_text(params: &ToolSchema, value: &str) -> String {