A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 20:00 UTC
[Rust Frontend] Add Phi-4 mini JSON tool parser (#44213)
Devin Lai committed
Jun 6, 2026 at 10:40 UTC
c8beda4cc38b1b34c172dc149b31b4e39dc2b569
6 files changed
+346
-4
rust/src/chat/src/lib.rs
+1
-1
@@ -234,7 +234,7 @@ mod tests {
234
)
235
.unwrap_err();
236
237
- expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, hermes, hy_v3, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, mistral, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
237
+ expect_test::expect!["tool parser `definitely_missing_tool_parser` is not registered (choose from: deepseek_v3, deepseek_v31, deepseek_v32, deepseek_v4, gemma4, glm45, glm47, hermes, hy_v3, internlm, kimi_k2, llama3_json, llama4_json, minimax_m2, mistral, phi4_mini_json, qwen3_coder, qwen3_xml)"].assert_eq(&error.to_report_string());
238
}
239
240
#[test]
rust/src/chat/src/parser/tool/mod.rs
+4
-2
@@ -6,8 +6,8 @@ pub use vllm_tool_parser::{
6
DeepSeekV3ToolParser, DeepSeekV4ToolParser, DeepSeekV31ToolParser, DeepSeekV32ToolParser,
7
Gemma4ToolParser, Glm45MoeToolParser, Glm47MoeToolParser, HermesToolParser, HyV3ToolParser,
8
Internlm2ToolParser, KimiK2ToolParser, Llama3JsonToolParser, MinimaxM2ToolParser,
9
- MistralToolParser, Qwen3CoderToolParser, Qwen3XmlToolParser, ToolCallDelta, ToolParser,
10
- ToolParserError, ToolParserOutput,
9
+ MistralToolParser, Phi4MiniJsonToolParser, Qwen3CoderToolParser, Qwen3XmlToolParser,
10
+ ToolCallDelta, ToolParser, ToolParserError, ToolParserOutput,
11
};
12
13
use crate::parser::ParserFactory;
@@ -32,6 +32,7 @@ pub mod names {
32
pub const LLAMA4_JSON: &str = "llama4_json";
33
pub const MINIMAX_M2: &str = "minimax_m2";
34
pub const MISTRAL: &str = "mistral";
35
+ pub const PHI4_MINI_JSON: &str = "phi4_mini_json";
36
pub const QWEN3_CODER: &str = "qwen3_coder";
37
pub const QWEN3_XML: &str = "qwen3_xml";
38
}
@@ -71,6 +72,7 @@ impl ToolParserFactory {
72
.register_parser::<Llama3JsonToolParser>(names::LLAMA4_JSON)
73
.register_parser::<MinimaxM2ToolParser>(names::MINIMAX_M2)
74
.register_parser::<MistralToolParser>(names::MISTRAL)
75
+ .register_parser::<Phi4MiniJsonToolParser>(names::PHI4_MINI_JSON)
76
.register_parser::<Qwen3XmlToolParser>(names::QWEN3_XML)
77
.register_parser::<Qwen3CoderToolParser>(names::QWEN3_CODER);
78
rust/src/chat/src/parser/tool/tests.rs
+11
@@ -191,3 +191,14 @@ fn factory_new_resolves_default_patterns() {
191
None
192
);
193
}
194
+
195
+#[test]
196
+fn factory_new_registers_phi4_mini_json_by_name() {
197
+ // phi-4-mini is registered by explicit name only (matching Python's
198
+ // `--tool-call-parser phi4_mini_json`); it is intentionally not mapped to
199
+ // any model-name pattern.
200
+ let factory = ToolParserFactory::new();
201
+
202
+ assert!(factory.contains(names::PHI4_MINI_JSON));
203
+ factory.create(names::PHI4_MINI_JSON, &[]).unwrap();
204
+}
rust/src/tool-parser/src/json/mod.rs
+2
@@ -4,12 +4,14 @@ pub use hermes::HermesToolParser;
4
pub use internlm2::Internlm2ToolParser;
5
pub use llama::Llama3JsonToolParser;
6
pub use mistral::MistralToolParser;
7
+pub use phi4mini::Phi4MiniJsonToolParser;
8
pub use qwen::Qwen3XmlToolParser;
9
10
mod hermes;
11
mod internlm2;
12
mod llama;
13
mod mistral;
14
+mod phi4mini;
15
mod qwen;
16
17
use winnow::ascii::multispace0 as ws0;
rust/src/tool-parser/src/json/phi4mini.rs
new
+327
@@ -0,0 +1,327 @@
1
+use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace};
2
+use crate::{Result, Tool, ToolParser, ToolParserOutput};
3
+
4
+const PHI4MINI_CONFIG: JsonToolCallConfig = JsonToolCallConfig {
5
+ parser_name: "Phi4Mini",
6
+ start_marker: "functools[",
7
+ end_marker: "]",
8
+ marker_whitespace: JsonToolCallWhitespace::Optional,
9
+ delimiter: Some(","),
10
+ name_key: "name",
11
+ // Accept both key variants emitted by Phi-4 Mini tool-call templates.
12
+ arguments_key: &["arguments", "parameters"],
13
+};
14
+
15
+/// Tool parser for phi-4-mini models.
16
+///
17
+/// Example tool-call content:
18
+///
19
+/// ```text
20
+/// functools[{"name": "get_weather", "arguments": {"location": "Tokyo"}}]
21
+/// ```
22
+///
23
+/// phi-4-mini emits an array of tool-call objects wrapped in a `functools[..]`
24
+/// envelope. Each object names the function with `name` and carries its
25
+/// arguments under `arguments` (preferred) or `parameters`. Arguments are
26
+/// already OpenAI-style JSON text, so they are streamed as raw argument deltas
27
+/// without schema conversion or JSON normalization.
28
+pub struct Phi4MiniJsonToolParser {
29
+ inner: JsonToolCallParser,
30
+}
31
+
32
+impl Phi4MiniJsonToolParser {
33
+ /// Create a phi-4-mini tool parser.
34
+ fn new(_tools: &[Tool]) -> Self {
35
+ Self {
36
+ inner: JsonToolCallParser::new(PHI4MINI_CONFIG),
37
+ }
38
+ }
39
+}
40
+
41
+impl ToolParser for Phi4MiniJsonToolParser {
42
+ /// Create a boxed phi-4-mini tool parser.
43
+ fn create(tools: &[Tool]) -> Result<Box<dyn ToolParser>>
44
+ where
45
+ Self: Sized + 'static,
46
+ {
47
+ Ok(Box::new(Self::new(tools)))
48
+ }
49
+
50
+ /// Feed one decoded text chunk through the phi-4-mini parser.
51
+ fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> {
52
+ self.inner.parse_into(chunk, output)
53
+ }
54
+
55
+ /// Flush any buffered partial state at end of stream.
56
+ fn finish(&mut self) -> Result<ToolParserOutput> {
57
+ self.inner.finish()
58
+ }
59
+
60
+ /// Clear parser state and return currently uncommitted buffered text.
61
+ fn reset(&mut self) -> String {
62
+ self.inner.reset()
63
+ }
64
+}
65
+
66
+#[cfg(test)]
67
+mod tests {
68
+ use expect_test::expect;
69
+ use thiserror_ext::AsReport;
70
+
71
+ use super::Phi4MiniJsonToolParser;
72
+ use crate::test_utils::{collect_stream, split_by_chars, test_tools};
73
+ use crate::{ToolParser, ToolParserTestExt as _};
74
+
75
+ /// Build one phi-4-mini tool-call object: `{"name":..,"<args_key>":<args>}`.
76
+ fn build_call(function_name: &str, args_key: &str, arguments: &str) -> String {
77
+ format!(r#"{{"name":"{function_name}","{args_key}":{arguments}}}"#)
78
+ }
79
+
80
+ /// Wrap tool-call objects in the `functools[..]` envelope.
81
+ fn wrap(calls: &[String]) -> String {
82
+ format!("functools[{}]", calls.join(","))
83
+ }
84
+
85
+ #[test]
86
+ fn phi4mini_parse_complete_without_tool_call_keeps_text() {
87
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
88
+ let result = parser.parse_complete("Hello, world!").unwrap();
89
+
90
+ assert_eq!(result.normal_text, "Hello, world!");
91
+ assert!(result.calls.is_empty());
92
+ }
93
+
94
+ #[test]
95
+ fn phi4mini_parse_complete_extracts_arguments_key() {
96
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
97
+ let arguments = r#"{"location":"Tokyo","days":"3"}"#;
98
+ let result = parser
99
+ .parse_complete(&wrap(&[build_call("get_weather", "arguments", arguments)]))
100
+ .unwrap();
101
+
102
+ assert_eq!(result.calls.len(), 1);
103
+ assert_eq!(result.calls[0].tool_index, 0);
104
+ assert_eq!(result.calls[0].name.as_deref(), Some("get_weather"));
105
+ assert_eq!(result.calls[0].arguments, arguments);
106
+ }
107
+
108
+ #[test]
109
+ fn phi4mini_parse_complete_falls_back_to_parameters_key() {
110
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
111
+ let arguments = r#"{"location":"Tokyo"}"#;
112
+ let result = parser
113
+ .parse_complete(&wrap(&[build_call("get_weather", "parameters", arguments)]))
114
+ .unwrap();
115
+
116
+ assert_eq!(result.calls.len(), 1);
117
+ assert_eq!(result.calls[0].name.as_deref(), Some("get_weather"));
118
+ assert_eq!(result.calls[0].arguments, arguments);
119
+ }
120
+
121
+ #[test]
122
+ fn phi4mini_extracts_multiple_comma_delimited_calls() {
123
+ let input = wrap(&[
124
+ build_call("get_weather", "arguments", r#"{"location":"Shanghai"}"#),
125
+ build_call("add", "arguments", r#"{"x":1,"y":2}"#),
126
+ ]);
127
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
128
+
129
+ let result = parser.parse_complete(&input).unwrap();
130
+
131
+ expect![[r#"
132
+ ToolParserOutput {
133
+ normal_text: "",
134
+ calls: [
135
+ ToolCallDelta {
136
+ tool_index: 0,
137
+ name: Some(
138
+ "get_weather",
139
+ ),
140
+ arguments: "{\"location\":\"Shanghai\"}",
141
+ },
142
+ ToolCallDelta {
143
+ tool_index: 1,
144
+ name: Some(
145
+ "add",
146
+ ),
147
+ arguments: "{\"x\":1,\"y\":2}",
148
+ },
149
+ ],
150
+ }
151
+ "#]]
152
+ .assert_debug_eq(&result);
153
+ }
154
+
155
+ /// The shared JSON core scans matched braces, so bracket-bearing argument
156
+ /// values are forwarded intact.
157
+ #[test]
158
+ fn phi4mini_array_valued_arguments_are_not_truncated() {
159
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
160
+ let arguments = r#"{"items":[1,2],"flag":true}"#;
161
+ let result = parser
162
+ .parse_complete(&wrap(&[build_call("convert", "arguments", arguments)]))
163
+ .unwrap();
164
+
165
+ assert_eq!(result.calls.len(), 1);
166
+ assert_eq!(result.calls[0].arguments, arguments);
167
+ }
168
+
169
+ /// Preface text before a tool call is preserved as normal_text, consistent
170
+ /// with the other JSON parsers in this crate.
171
+ #[test]
172
+ fn phi4mini_preserves_text_before_tool_call() {
173
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
174
+ let input = format!(
175
+ "Let me check.\n{}",
176
+ wrap(&[build_call(
177
+ "get_weather",
178
+ "arguments",
179
+ r#"{"location":"Tokyo"}"#
180
+ )])
181
+ );
182
+
183
+ let result = parser.parse_complete(&input).unwrap();
184
+
185
+ assert_eq!(result.normal_text, "Let me check.\n");
186
+ assert_eq!(result.calls.len(), 1);
187
+ }
188
+
189
+ #[test]
190
+ fn phi4mini_does_not_validate_or_normalize_arguments() {
191
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
192
+ let arguments = r#"{"location":"Tokyo",}"#;
193
+ let result = parser
194
+ .parse_complete(&wrap(&[build_call("get_weather", "arguments", arguments)]))
195
+ .unwrap();
196
+
197
+ assert_eq!(result.calls[0].arguments, arguments);
198
+ }
199
+
200
+ /// The bundled `tool_chat_template_phi4_mini.jinja` emits objects with
201
+ /// whitespace after `:` and `,` (e.g. `{"name": "f", "arguments": {..}}`).
202
+ /// Confirm the parser handles that real model format and preserves the
203
+ /// inner argument spacing verbatim.
204
+ #[test]
205
+ fn phi4mini_accepts_real_model_whitespace_format() {
206
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
207
+ let input = r#"functools[{"name": "get_weather", "arguments": {"location": "Tokyo"}}]"#;
208
+
209
+ let result = parser.parse_complete(input).unwrap();
210
+
211
+ assert_eq!(result.calls.len(), 1);
212
+ assert_eq!(result.calls[0].name.as_deref(), Some("get_weather"));
213
+ assert_eq!(result.calls[0].arguments, r#"{"location": "Tokyo"}"#);
214
+ }
215
+
216
+ /// Argument deltas are streamed through the shared JSON core.
217
+ #[test]
218
+ fn phi4mini_streaming_emits_argument_deltas() {
219
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
220
+ let chunks = [
221
+ "preface functo",
222
+ "ols[",
223
+ r#"{"name":"get_weather","arguments":"#,
224
+ r#"{"location":"#,
225
+ r#""Beijing""#,
226
+ r#"}"#,
227
+ r#"}]"#,
228
+ " suffix",
229
+ ];
230
+
231
+ let result = collect_stream(&mut parser, &chunks);
232
+
233
+ assert_eq!(result.normal_text, "preface suffix");
234
+ assert_eq!(result.calls.len(), 1);
235
+ assert_eq!(result.calls[0].name.as_deref(), Some("get_weather"));
236
+ assert_eq!(result.calls[0].arguments, r#"{"location":"Beijing"}"#);
237
+ }
238
+
239
+ #[test]
240
+ fn phi4mini_streaming_handles_split_markers() {
241
+ let input = format!(
242
+ "hello {}",
243
+ wrap(&[build_call(
244
+ "get_weather",
245
+ "arguments",
246
+ r#"{"location":"Tokyo"}"#
247
+ )])
248
+ );
249
+ let chunks = split_by_chars(&input, 5);
250
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
251
+
252
+ let result = collect_stream(&mut parser, &chunks);
253
+
254
+ assert_eq!(result.normal_text, "hello ");
255
+ assert_eq!(result.calls.len(), 1);
256
+ assert_eq!(result.calls[0].arguments, r#"{"location":"Tokyo"}"#);
257
+ }
258
+
259
+ #[test]
260
+ fn phi4mini_finish_errors_on_truncated_tool_call() {
261
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
262
+ let _ = parser
263
+ .parse_chunk(r#"functools[{"name":"get_weather","arguments":{"location""#)
264
+ .unwrap();
265
+ let error = parser.finish().unwrap_err();
266
+
267
+ assert!(
268
+ error.to_report_string().contains("incomplete Phi4Mini tool call"),
269
+ "finish() reports the truncated tool call as incomplete: {}",
270
+ error.to_report_string(),
271
+ );
272
+ }
273
+
274
+ #[test]
275
+ fn phi4mini_preserve_special_tokens_is_false() {
276
+ let parser = Phi4MiniJsonToolParser::new(&test_tools());
277
+ assert!(!parser.preserve_special_tokens());
278
+ }
279
+
280
+ /// The brace-scanning core handles nested arrays and objects in arguments.
281
+ #[test]
282
+ fn phi4mini_parses_nested_arrays_and_objects() {
283
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
284
+ let arguments =
285
+ r#"{"array_field":["a","b","c"],"object_field":{"nested":"value"},"empty_object":{}}"#;
286
+ let result = parser
287
+ .parse_complete(&wrap(&[build_call("convert", "arguments", arguments)]))
288
+ .unwrap();
289
+
290
+ assert_eq!(result.calls.len(), 1);
291
+ assert_eq!(result.calls[0].name.as_deref(), Some("convert"));
292
+ assert_eq!(result.calls[0].arguments, arguments);
293
+ }
294
+
295
+ /// The chat template emits parallel calls as `},\n {` (comma + newline +
296
+ /// indent). Confirm the `Optional` marker whitespace and `,` delimiter
297
+ /// parse the real multi-call layout.
298
+ #[test]
299
+ fn phi4mini_parses_parallel_calls_in_template_format() {
300
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
301
+ let input = concat!(
302
+ "functools[\n",
303
+ " {\"name\": \"get_weather\", \"arguments\": {\"city\": \"Tokyo\"}},\n",
304
+ " {\"name\": \"add\", \"arguments\": {\"x\": 1, \"y\": 2}}\n",
305
+ "]"
306
+ );
307
+
308
+ let result = parser.parse_complete(input).unwrap();
309
+
310
+ assert_eq!(result.calls.len(), 2);
311
+ assert_eq!(result.calls[0].name.as_deref(), Some("get_weather"));
312
+ assert_eq!(result.calls[1].name.as_deref(), Some("add"));
313
+ }
314
+
315
+ /// The shared core requires an object after the start marker.
316
+ #[test]
317
+ fn phi4mini_empty_array_errors() {
318
+ let mut parser = Phi4MiniJsonToolParser::new(&test_tools());
319
+ let error = parser.parse_complete("functools[]").unwrap_err();
320
+
321
+ assert!(
322
+ error.to_report_string().contains("invalid Phi4Mini"),
323
+ "empty functools[] should error: {}",
324
+ error.to_report_string(),
325
+ );
326
+ }
327
+}
rust/src/tool-parser/src/lib.rs
+1
-1
@@ -26,7 +26,7 @@ pub use glm_xml::{Glm45MoeToolParser, Glm47MoeToolParser};
26
pub use hy_v3::HyV3ToolParser;
27
pub use json::{
28
HermesToolParser, Internlm2ToolParser, Llama3JsonToolParser, MistralToolParser,
29
- Qwen3XmlToolParser,
29
+ Phi4MiniJsonToolParser, Qwen3XmlToolParser,
30
};
31
pub use kimi_k2::KimiK2ToolParser;
32
pub use minimax_m2::MinimaxM2ToolParser;