@tduncan / Nvidia-Optimization-Stack-VLLM / commits / f1e921b6d6

[Rust Frontend] Support dynamic tools from developer messages (#51144)

Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Bugen Zhao committed Aug 11, 2026 at 11:30 UTC f1e921b6d6404ad1debd14ec0275b007c8c2ee11
28 files changed +686 -258
rust/src/chat/src/error.rs
+10 -1
@@ -71,6 +71,12 @@ pub enum Error {
71 StreamClosedBeforeTerminalOutput { request_id: String },
72 #[error("tool call stream state is inconsistent: {message}")]
73 ToolCallStreamInvariant { message: String },
74 + #[error("duplicate tool name `{name}`")]
75 + DuplicateToolName { name: String },
76 + #[error("tool_choice requires at least one available tool")]
77 + ToolChoiceRequiresTools,
78 + #[error("tool_choice function `{name}` was not found in the available tools")]
79 + ToolChoiceFunctionNotFound { name: String },
80 #[error("failed to build structural tag: {message}")]
81 StructuralTag { message: String },
82 #[error(transparent)]
@@ -85,7 +91,10 @@ impl Error {
91 /// Whether this error represents invalid user request parameters.
92 pub fn is_request_validation_error(&self) -> bool {
93 match self {
88 - Self::PromptTooLong { .. } => true,
94 + Self::PromptTooLong { .. }
95 + | Self::DuplicateToolName { .. }
96 + | Self::ToolChoiceRequiresTools
97 + | Self::ToolChoiceFunctionNotFound { .. } => true,
98 Self::Text(error) => error.is_request_validation_error(),
99 Self::UnsupportedMultimodalRenderer
100 | Self::UnsupportedMultimodalContent(_)
rust/src/chat/src/lib.rs
+1 -1
@@ -39,7 +39,7 @@ pub use renderer::{
39 };
40 pub use request::{
41 ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
42 - ChatToolChoice, GenerationPromptMode, ReasoningEffort, SamplingParams,
42 + ChatToolChoice, GenerationPromptMode, ReasoningEffort, ResolvedToolContext, SamplingParams,
43 };
44 pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
45 pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
rust/src/chat/src/output/default/mod.rs
+3 -3
@@ -52,7 +52,7 @@ impl DefaultChatOutputProcessor {
52 ) -> ChatResult<Self> {
53 let parser = if tool_call_parser == reasoning_parser
54 && let Some(parser) = Self::resolve_optional_unified_parser(
55 - &request.tools,
55 + request.tools(),
56 model_id,
57 tokenizer.clone(),
58 tool_call_parser,
@@ -62,7 +62,7 @@ impl DefaultChatOutputProcessor {
62 let tool_parsing_enabled = request.tool_parsing_enabled();
63 let tool_parser = if tool_parsing_enabled {
64 Some(Self::resolve_tool_parser(
65 - &request.tools,
65 + request.tools(),
66 model_id,
67 tool_call_parser,
68 )?)
@@ -82,7 +82,7 @@ impl DefaultChatOutputProcessor {
82
83 Ok(Self {
84 parser,
85 - parallel_tool_calls: request.parallel_tool_calls,
85 + parallel_tool_calls: request.parallel_tool_calls(),
86 })
87 }
88
rust/src/chat/src/output/default/structural_tag.rs
+66 -14
@@ -31,7 +31,7 @@ pub(super) fn apply_structural_tag_constraint(
31 };
32
33 let tools = request
34 - .tools
34 + .tools()
35 .iter()
36 .map(|tool| {
37 ToolParam::Function(FunctionToolParam::new(FunctionDefinition {
@@ -67,13 +67,13 @@ pub(super) fn apply_structural_tag_constraint(
67 ///
68 /// Returns `None` if no structural tag constraints should be applied.
69 fn structural_tag_tool_choice(request: &ChatRequest) -> Option<StructuralTagToolChoice> {
70 - if request.tools.is_empty() {
70 + if request.tools().is_empty() {
71 return None;
72 }
73
74 - match &request.tool_choice {
74 + match request.tool_choice() {
75 // For `Auto`, only apply the structural tag if there's at least one strict tool.
76 - ChatToolChoice::Auto if request.tools.iter().any(|tool| tool.strict == Some(true)) => {
76 + ChatToolChoice::Auto if request.tools().iter().any(|tool| tool.strict == Some(true)) => {
77 Some(StructuralTagToolChoice::auto())
78 }
79 ChatToolChoice::Auto | ChatToolChoice::None => None,
@@ -92,6 +92,7 @@ mod tests {
92 use vllm_parser::tool::{Qwen3CoderToolParser, Tool, ToolParser};
93
94 use super::*;
95 + use crate::request::{ChatMessage, ResolvedToolContext};
96
97 fn chat_tool(name: &str, strict: Option<bool>) -> Tool {
98 Tool {
@@ -114,8 +115,8 @@ mod tests {
115
116 fn request(tool_choice: ChatToolChoice, tools: Vec<Tool>) -> ChatRequest {
117 ChatRequest {
117 - tool_choice,
118 - tools,
118 + tool_context: ResolvedToolContext::new(&[], tools, Some(tool_choice), true)
119 + .expect("tool context should resolve"),
120 ..ChatRequest::for_test()
121 }
122 }
@@ -145,7 +146,7 @@ mod tests {
146 #[test]
147 fn auto_strict_tool_choice_builds_structural_tag() {
148 let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]);
148 - let parser = qwen3_coder_parser(&request.tools);
149 + let parser = qwen3_coder_parser(request.tools());
150
151 apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
152 .expect("structural tag should build");
@@ -155,10 +156,61 @@ mod tests {
156 assert!(tag.to_string().contains("search"));
157 }
158
159 + #[test]
160 + fn required_dynamic_tool_builds_structural_tag_from_effective_tools() {
161 + let messages = vec![ChatMessage::developer(
162 + "",
163 + Some(vec![chat_tool("lookup", None)]),
164 + )];
165 + let tool_context =
166 + ResolvedToolContext::new(&messages, Vec::new(), Some(ChatToolChoice::Required), true)
167 + .expect("dynamic tool context should resolve");
168 + let mut request = ChatRequest {
169 + messages,
170 + tool_context,
171 + ..ChatRequest::for_test()
172 + };
173 + let parser = qwen3_coder_parser(request.tools());
174 +
175 + apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
176 + .expect("structural tag should build");
177 +
178 + let tag = structural_tag_value(&request);
179 + assert!(tag.to_string().contains("lookup"));
180 + }
181 +
182 + #[test]
183 + fn required_initial_and_dynamic_tools_build_one_structural_tag() {
184 + let messages = vec![ChatMessage::developer(
185 + "",
186 + Some(vec![chat_tool("lookup", None)]),
187 + )];
188 + let tool_context = ResolvedToolContext::new(
189 + &messages,
190 + vec![chat_tool("search", None)],
191 + Some(ChatToolChoice::Required),
192 + true,
193 + )
194 + .expect("tool context should resolve");
195 + let mut request = ChatRequest {
196 + messages,
197 + tool_context,
198 + ..ChatRequest::for_test()
199 + };
200 + let parser = qwen3_coder_parser(request.tools());
201 +
202 + apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
203 + .expect("structural tag should build");
204 +
205 + let tag = structural_tag_value(&request).to_string();
206 + assert!(tag.contains("search"));
207 + assert!(tag.contains("lookup"));
208 + }
209 +
210 #[test]
211 fn auto_non_strict_tool_choice_skips_structural_tag() {
212 let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]);
161 - let parser = qwen3_coder_parser(&request.tools);
213 + let parser = qwen3_coder_parser(request.tools());
214
215 apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
216 .expect("structural tag decision should succeed");
@@ -173,7 +225,7 @@ mod tests {
225 backend: StructuredOutputBackend::Xgrammar,
226 ..StructuredOutputsParams::json(json!({"type": "object"}))
227 });
176 - let parser = qwen3_coder_parser(&request.tools);
228 + let parser = qwen3_coder_parser(request.tools());
229
230 apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
231 .expect("structural tag should build");
@@ -188,7 +240,7 @@ mod tests {
240 #[test]
241 fn required_tool_choice_builds_structural_tag_without_strict_tools() {
242 let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]);
191 - let parser = qwen3_coder_parser(&request.tools);
243 + let parser = qwen3_coder_parser(request.tools());
244
245 apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
246 .expect("structural tag should build");
@@ -205,7 +257,7 @@ mod tests {
257 backend: StructuredOutputBackend::Xgrammar,
258 ..StructuredOutputsParams::json_object()
259 });
208 - let parser = qwen3_coder_parser(&request.tools);
260 + let parser = qwen3_coder_parser(request.tools());
261
262 apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
263 .expect("structural tag should build");
@@ -225,7 +277,7 @@ mod tests {
277 },
278 vec![chat_tool("search", None), chat_tool("lookup", None)],
279 );
228 - let parser = qwen3_coder_parser(&request.tools);
280 + let parser = qwen3_coder_parser(request.tools());
281
282 apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
283 .expect("structural tag should build");
@@ -238,7 +290,7 @@ mod tests {
290 #[test]
291 fn none_tool_choice_skips_structural_tag() {
292 let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]);
241 - let parser = qwen3_coder_parser(&request.tools);
293 + let parser = qwen3_coder_parser(request.tools());
294
295 apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
296 .expect("structural tag decision should succeed");
@@ -253,7 +305,7 @@ mod tests {
305 backend: StructuredOutputBackend::Xgrammar,
306 ..StructuredOutputsParams::json_object()
307 });
256 - let parser = qwen3_coder_parser(&request.tools);
308 + let parser = qwen3_coder_parser(request.tools());
309
310 apply_structural_tag_constraint(&mut request, parser.structural_tag_builder())
311 .expect("structural tag decision should succeed");
rust/src/chat/src/output/harmony/mod.rs
+1 -1
@@ -75,7 +75,7 @@ impl HarmonyChatOutputProcessor {
75 Ok(Self {
76 encoding: harmony_encoding()?,
77 tool_calls_enabled: request.tool_parsing_enabled(),
78 - parallel_tool_calls: request.parallel_tool_calls,
78 + parallel_tool_calls: request.parallel_tool_calls(),
79 })
80 }
81 }
rust/src/chat/src/output/harmony/tests.rs
+13 -12
@@ -10,7 +10,7 @@ use vllm_text::output::{DecodedLogprobs, DecodedPositionLogprobs, DecodedTextEve
10
11 use super::*;
12 use crate::output::ChatOutputProcessor;
13 -use crate::request::{ChatRequest, ChatTool, ChatToolChoice};
13 +use crate::request::{ChatRequest, ChatTool, ChatToolChoice, ResolvedToolContext};
14 use crate::{AssistantMessageExt, ChatEvent, FinishReason};
15
16 fn assistant_prefix() -> Vec<u32> {
@@ -72,18 +72,19 @@ async fn collect_events(
72 }
73
74 fn request_with_tools() -> ChatRequest {
75 + let tools = vec![ChatTool {
76 + name: "get_weather".to_string(),
77 + description: Some("Get weather".to_string()),
78 + parameters: serde_json::json!({
79 + "type": "object",
80 + "properties": {"city": {"type": "string"}},
81 + "required": ["city"]
82 + }),
83 + strict: None,
84 + }];
85 ChatRequest {
76 - tool_choice: ChatToolChoice::Auto,
77 - tools: vec![ChatTool {
78 - name: "get_weather".to_string(),
79 - description: Some("Get weather".to_string()),
80 - parameters: serde_json::json!({
81 - "type": "object",
82 - "properties": {"city": {"type": "string"}},
83 - "required": ["city"]
84 - }),
85 - strict: None,
86 - }],
86 + tool_context: ResolvedToolContext::new(&[], tools, Some(ChatToolChoice::Auto), true)
87 + .expect("tool context should resolve"),
88 ..ChatRequest::for_test()
89 }
90 }
rust/src/chat/src/renderer/deepseek_v32/encoding.rs
+11 -3
@@ -2,6 +2,9 @@
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 //! DeepSeek V3.2 prompt renderer.
5 +//!
6 +//! Official Python reference:
7 +//! <https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/encoding/encoding_dsv32.py>
8
9 use std::collections::{HashMap, HashSet};
10 use std::fmt::Write as _;
@@ -48,15 +51,20 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
51 request.messages.last().map(ChatMessage::role),
52 Some(ChatRole::User | ChatRole::Developer)
53 );
51 - let render_offset = isize::from(request.tool_parsing_enabled());
54 + let initial_tools = if request.tool_parsing_enabled() {
55 + request.initial_tools()
56 + } else {
57 + &[]
58 + };
59 + let render_offset = isize::from(!initial_tools.is_empty());
60 let last_user_render_index =
61 find_last_user_render_index(request.messages.as_slice(), render_offset);
62 let last_user_actual_index = find_last_user_actual_index(request.messages.as_slice());
63 let continue_final_message = request.chat_options.continue_final_message();
64 let mut prompt = String::from(BOS_TOKEN);
65
58 - if request.tool_parsing_enabled() {
59 - render_system_message(&mut prompt, None, &request.tools)?;
66 + if !initial_tools.is_empty() {
67 + render_system_message(&mut prompt, None, initial_tools)?;
68 }
69
70 for (message_index, message) in request.messages.iter().enumerate() {
rust/src/chat/src/renderer/deepseek_v32/fixtures/test_input_developer_tools.json new
+32
@@ -0,0 +1,32 @@
1 +{
2 + "messages": [
3 + {
4 + "role": "user",
5 + "content": "question"
6 + },
7 + {
8 + "role": "developer",
9 + "content": "policy",
10 + "tools": [
11 + {
12 + "type": "function",
13 + "function": {
14 + "name": "lookup",
15 + "parameters": {
16 + "type": "object"
17 + }
18 + }
19 + }
20 + ]
21 + },
22 + {
23 + "role": "assistant",
24 + "reasoning_content": "thought",
25 + "content": "answer"
26 + }
27 + ],
28 + "add_generation_prompt": false,
29 + "template_kwargs": {
30 + "thinking": true
31 + }
32 +}
rust/src/chat/src/renderer/deepseek_v32/fixtures/test_output_developer_tools.txt new
+37
@@ -0,0 +1,37 @@
1 +<｜begin▁of▁sentence｜><｜User｜>question<｜Assistant｜></think><｜User｜>
2 +
3 +## Tools
4 +
5 +You have access to a set of tools you can use to answer the user's question.
6 +You can invoke functions by writing a "<｜DSML｜function_calls>" block like the following as part of your reply to the user:
7 +<｜DSML｜function_calls>
8 +<｜DSML｜invoke name="$FUNCTION_NAME">
9 +<｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</｜DSML｜parameter>
10 +...
11 +</｜DSML｜invoke>
12 +<｜DSML｜invoke name="$FUNCTION_NAME2">
13 +...
14 +</｜DSML｜invoke>
15 +</｜DSML｜function_calls>
16 +
17 +String and scalar parameters should be specified as is without any escaping or quotes, while lists and objects should use JSON format. The "string" attribute should be set to "true" for string type parameters and "false" for other types (numbers, booleans, arrays, objects).
18 +
19 +If the thinking_mode is enabled, then after function results you should strongly consider outputting a thinking block. Here is an example:
20 +
21 +<｜DSML｜function_calls>
22 +...
23 +</｜DSML｜function_calls>
24 +
25 +<function_results>
26 +...
27 +</function_results>
28 +
29 +<think>...thinking about results</think>
30 +
31 +Here are the functions available in JSONSchema format:
32 +<functions>
33 +{"name": "lookup", "parameters": {"type": "object"}}
34 +</functions>
35 +
36 +
37 +# The user's message is: policy<｜Assistant｜><think>thought</think>answer<｜end▁of▁sentence｜>
\ No newline at end of file
rust/src/chat/src/renderer/deepseek_v32/tests.rs
+30 -19
@@ -13,6 +13,7 @@ use crate::event::{AssistantContentBlock, AssistantToolCall};
13 use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
14 use crate::request::{
15 ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice, GenerationPromptMode,
16 + ResolvedToolContext,
17 };
18 use crate::{ChatRenderer, ChatRole};
19
@@ -85,6 +86,14 @@ fn renders_vllm_parity_prompt_for_request_level_tools_fixture() {
86 );
87 }
88
89 +#[test]
90 +fn renders_developer_tools_like_hf_python() {
91 + assert_fixture(
92 + "test_input_developer_tools.json",
93 + expect_file!["fixtures/test_output_developer_tools.txt"],
94 + );
95 +}
96 +
97 #[test]
98 fn renders_official_search_fixture_without_date() {
99 assert_fixture(
@@ -103,27 +112,29 @@ fn renders_official_search_fixture_with_date() {
112
113 #[test]
114 fn request_level_tools_are_lowered_as_synthetic_leading_system_message() {
115 + let messages = vec![
116 + ChatMessage::system("System prompt."),
117 + ChatMessage::text(ChatRole::User, "Hello"),
118 + ];
119 + let tools = vec![ChatTool {
120 + name: "lookup".to_string(),
121 + description: Some("Look things up".to_string()),
122 + parameters: json!({
123 + "type": "object",
124 + "properties": {
125 + "query": {
126 + "type": "string"
127 + }
128 + },
129 + "required": ["query"]
130 + }),
131 + strict: None,
132 + }];
133 let mut request = ChatRequest {
134 request_id: "deepseek-v32-tools".to_string(),
108 - messages: vec![
109 - ChatMessage::system("System prompt."),
110 - ChatMessage::text(ChatRole::User, "Hello"),
111 - ],
112 - tools: vec![ChatTool {
113 - name: "lookup".to_string(),
114 - description: Some("Look things up".to_string()),
115 - parameters: json!({
116 - "type": "object",
117 - "properties": {
118 - "query": {
119 - "type": "string"
120 - }
121 - },
122 - "required": ["query"]
123 - }),
124 - strict: None,
125 - }],
126 - tool_choice: ChatToolChoice::Auto,
135 + tool_context: ResolvedToolContext::new(&messages, tools, Some(ChatToolChoice::Auto), true)
136 + .expect("tool context should resolve"),
137 + messages,
138 ..ChatRequest::for_test()
139 };
140 request
rust/src/chat/src/renderer/deepseek_v4/encoding.rs
+3 -3
@@ -3,8 +3,8 @@
3
4 //! DeepSeek V4 prompt renderer.
5 //!
6 -//! Original Python implementation:
7 -//! <https://github.com/vllm-project/vllm/blob/main/vllm/tokenizers/deepseek_v4_encoding.py>
6 +//! Official Python reference:
7 +//! <https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash/blob/main/encoding/encoding_dsv4.py>
8
9 use std::collections::HashMap;
10 use std::fmt::Write as _;
@@ -156,7 +156,7 @@ fn resolve_thinking_options(request: &ChatRequest) -> Result<(ThinkingMode, &'st
156 /// Return request-level tools only when native tool parsing is enabled.
157 fn request_tools(request: &ChatRequest) -> &[ChatTool] {
158 if request.tool_parsing_enabled() {
159 - request.tools.as_slice()
159 + request.initial_tools()
160 } else {
161 &[]
162 }
rust/src/chat/src/renderer/deepseek_v4/fixtures/test_input_developer_tools.json new
+31
@@ -0,0 +1,31 @@
1 +{
2 + "messages": [
3 + {
4 + "role": "user",
5 + "content": "question"
6 + },
7 + {
8 + "role": "developer",
9 + "content": "policy",
10 + "tools": [
11 + {
12 + "type": "function",
13 + "function": {
14 + "name": "lookup",
15 + "parameters": {
16 + "type": "object"
17 + }
18 + }
19 + }
20 + ]
21 + },
22 + {
23 + "role": "assistant",
24 + "content": "answer"
25 + }
26 + ],
27 + "add_generation_prompt": false,
28 + "template_kwargs": {
29 + "thinking": true
30 + }
31 +}
rust/src/chat/src/renderer/deepseek_v4/fixtures/test_output_developer_tools.txt new
+28
@@ -0,0 +1,28 @@
1 +<｜begin▁of▁sentence｜><｜User｜>question<｜User｜>policy
2 +
3 +## Tools
4 +
5 +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜tool_calls>" block like the following:
6 +
7 +<｜DSML｜tool_calls>
8 +<｜DSML｜invoke name="$TOOL_NAME">
9 +<｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</｜DSML｜parameter>
10 +...
11 +</｜DSML｜invoke>
12 +<｜DSML｜invoke name="$TOOL_NAME2">
13 +...
14 +</｜DSML｜invoke>
15 +</｜DSML｜tool_calls>
16 +
17 +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
18 +
19 +If thinking_mode is enabled (triggered by <think>), you MUST output your complete reasoning inside <think>...</think> BEFORE any tool calls or final response.
20 +
21 +Otherwise, output directly after </think> with tool calls or final response.
22 +
23 +### Available Tool Schemas
24 +
25 +{"name": "lookup", "parameters": {"type": "object"}}
26 +
27 +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
28 +<｜Assistant｜><think></think>answer<｜end▁of▁sentence｜>
\ No newline at end of file
rust/src/chat/src/renderer/deepseek_v4/tests.rs
+8
@@ -63,6 +63,14 @@ fn renders_v4_fixture_2_multi_turn_drop_thinking() {
63 );
64 }
65
66 +#[test]
67 +fn renders_developer_tools_like_hf_python() {
68 + assert_fixture(
69 + "test_input_developer_tools.json",
70 + expect_file!["fixtures/test_output_developer_tools.txt"],
71 + );
72 +}
73 +
74 #[test]
75 fn reasoning_effort_max_adds_prefix_when_thinking_is_enabled() {
76 let mut request = ChatRequest {
rust/src/chat/src/renderer/harmony/mod.rs
+1 -1
@@ -216,7 +216,7 @@ fn preamble_tool_descriptions(
216 ) -> Vec<ToolDescription> {
217 let mut tools = Vec::new();
218 if request.tool_parsing_enabled() {
219 - tools.extend(to_tool_descriptions(&request.tools));
219 + tools.extend(to_tool_descriptions(request.initial_tools()));
220 }
221 if let Some(leading_developer_tools) = leading_developer_tools {
222 tools.extend(to_tool_descriptions(leading_developer_tools));
rust/src/chat/src/renderer/hf/mod.rs
+46 -5
@@ -179,7 +179,7 @@ impl HfChatRenderer {
179 None
180 };
181
182 - let tools = request.tool_parsing_enabled().then(|| to_template_tools(&request.tools));
182 + let tools = request.tool_parsing_enabled().then(|| to_template_tools(request.tools()));
183 trace!(
184 message_count = messages.len(),
185 content_format = ?effective_template.content_format(),
@@ -579,7 +579,7 @@ mod tests {
579 use super::{ChatTemplateContentFormatOption, HfChatRenderer, MultimodalRenderInfo};
580 use crate::request::{
581 ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice,
582 - GenerationPromptMode, ReasoningEffort,
582 + GenerationPromptMode, ReasoningEffort, ResolvedToolContext,
583 };
584 use crate::{AssistantContentBlock, ChatRenderer, Error, Result};
585
@@ -587,13 +587,48 @@ mod tests {
587 const QWEN3_5_0_8B_TEMPLATE: &str = include_str!("../../../tests/templates/qwen35.jinja");
588
589 fn sample_request(messages: Vec<ChatMessage>) -> ChatRequest {
590 + let tool_context = ResolvedToolContext::new(&messages, Vec::new(), None, true).unwrap();
591 ChatRequest {
592 messages,
593 + tool_context,
594 request_id: "render-test".to_string(),
595 ..ChatRequest::for_test()
596 }
597 }
598
599 + #[test]
600 + fn dynamic_tools_are_exposed_as_effective_template_tools() {
601 + let messages = vec![
602 + ChatMessage::developer(
603 + "",
604 + Some(vec![ChatTool {
605 + name: "lookup".to_string(),
606 + description: None,
607 + parameters: serde_json::json!({"type": "object"}),
608 + strict: None,
609 + }]),
610 + ),
611 + ChatMessage::user("hello"),
612 + ];
613 + let tool_context = ResolvedToolContext::new(&messages, Vec::new(), None, true).unwrap();
614 + let request = ChatRequest {
615 + messages,
616 + tool_context,
617 + request_id: "render-test".to_string(),
618 + ..ChatRequest::for_test()
619 + };
620 +
621 + let rendered = render(
622 + Some("{{ messages|length }}:{{ messages[0].role }}:{{ messages[0].tools[0].function.name }}:{{ tools[0].function.name }}"),
623 + &request,
624 + )
625 + .unwrap();
626 +
627 + assert_eq!(rendered, "2:developer:lookup:lookup");
628 + assert_eq!(request.tool_choice(), &ChatToolChoice::Auto);
629 + assert_eq!(request.tools().len(), 1);
630 + }
631 +
632 fn render(template: Option<&str>, request: &ChatRequest) -> Result<String> {
633 HfChatRenderer::new(
634 template.map(str::to_owned),
@@ -897,7 +932,7 @@ mod tests {
932 }
933
934 #[test]
900 - fn chat_template_exposes_developer_tools() {
935 + fn chat_template_preserves_developer_tools() {
936 let request = sample_request(vec![ChatMessage::developer(
937 "policy",
938 Some(vec![ChatTool {
@@ -1218,7 +1253,7 @@ mod tests {
1253 #[test]
1254 fn chat_template_exposes_tools_to_templates_when_auto_enabled() {
1255 let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
1221 - request.tools = vec![ChatTool {
1256 + let tools = vec![ChatTool {
1257 name: "get_weather".to_string(),
1258 description: Some("Get weather".to_string()),
1259 parameters: serde_json::json!({
@@ -1228,7 +1263,13 @@ mod tests {
1263 }),
1264 strict: None,
1265 }];
1231 - request.tool_choice = ChatToolChoice::Auto;
1266 + request.tool_context = crate::request::ResolvedToolContext::new(
1267 + &request.messages,
1268 + tools,
1269 + Some(ChatToolChoice::Auto),
1270 + true,
1271 + )
1272 + .expect("tool context should resolve");
1273
1274 let rendered = render(
1275 Some("{{ tools[0].function.name }}|{{ tools[0].function.parameters.required[0] }}"),
rust/src/chat/src/renderer/inkling/mod.rs
+1 -12
@@ -383,18 +383,7 @@ fn rendered_tools(request: &ChatRequest) -> Vec<&ChatTool> {
383 return Vec::new();
384 }
385
386 - let mut tools = Vec::with_capacity(request.tools.len());
387 - tools.extend(request.tools.iter());
388 - for message in &request.messages {
389 - if let ChatMessage::Developer {
390 - tools: Some(local_tools),
391 - ..
392 - } = message
393 - {
394 - tools.extend(local_tools.iter());
395 - }
396 - }
397 - tools
386 + request.tools().iter().collect()
387 }
388
389 fn tool_call_json(tool_call: &AssistantToolCall) -> Result<String> {
rust/src/chat/src/renderer/kimi_k3/encoding.rs
+3 -3
@@ -191,7 +191,7 @@ pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -
191 }
192 flush_tool_run(&mut out, &mut pending_tool_run, &tool_call_id_index)?;
193
194 - match &request.tool_choice {
194 + match request.tool_choice() {
195 ChatToolChoice::Required => {
196 write_internal_system(
197 &mut out,
@@ -202,7 +202,7 @@ pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -
202 }
203 // Emit only when tools are present: Rust defaults tool_choice to None
204 // for tool-free requests, which must not inject a tool-choice message.
205 - ChatToolChoice::None if !request.tools.is_empty() => {
205 + ChatToolChoice::None if !request.tools().is_empty() => {
206 write_internal_system(
207 &mut out,
208 "tool-choice",
@@ -226,7 +226,7 @@ pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -
226 fn request_tools(request: &ChatRequest) -> &[ChatTool] {
227 // Declare tools whenever the request carries them. K3 tool-declare is
228 // independent of tool_choice; tool_choice only injects control messages.
229 - request.tools.as_slice()
229 + request.initial_tools()
230 }
231
232 fn thinking_enabled(request: &ChatRequest) -> Result<bool> {
rust/src/chat/src/renderer/kimi_k3/fixtures/dynamic_only_tool_declare_input.json new
+40
@@ -0,0 +1,40 @@
1 +{
2 + "messages": [
3 + {
4 + "role": "user",
5 + "content": "Load the calculator."
6 + },
7 + {
8 + "role": "developer",
9 + "tools": [
10 + {
11 + "type": "function",
12 + "function": {
13 + "name": "calc",
14 + "description": "Evaluate an expression",
15 + "parameters": {
16 + "type": "object",
17 + "properties": {
18 + "expression": {
19 + "type": "string"
20 + }
21 + },
22 + "required": [
23 + "expression"
24 + ]
25 + }
26 + }
27 + }
28 + ]
29 + },
30 + {
31 + "role": "user",
32 + "content": "What is 2 + 2?"
33 + }
34 + ],
35 + "add_generation_prompt": true,
36 + "tool_choice": "none",
37 + "template_kwargs": {
38 + "thinking": true
39 + }
40 +}
rust/src/chat/src/renderer/kimi_k3/fixtures/dynamic_only_tool_declare_output.txt new
+10
@@ -0,0 +1,10 @@
1 +<|open|>message role="system" type="thinking-effort"<|sep|>`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.
2 +Now the system is invoked with `thinking_effort=max`.<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>Load the calculator.<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="tool-declare"<|sep|>## New Tools Available
3 +The system dynamically extends the toolset via lazy-loading.
4 +You have access to all existing and extended tools.
5 +Here are the specs for the extended tools.
6 +
7 +```json
8 +[{"function":{"description":"Evaluate an expression","name":"calc","parameters":{"properties":{"expression":{"type":"string"}},"required":["expression"],"type":"object"}},"type":"function"}]
9 +```<|close|>message<|sep|><|end_of_msg|><|open|>message role="user"<|sep|>What is 2 + 2?<|close|>message<|sep|><|end_of_msg|><|open|>message role="system" type="tool-choice"<|sep|>The system is invoked with `tool_choice=none`.
10 +You MUST NOT call any tools in the next message.<|close|>message<|sep|><|end_of_msg|><|open|>message role="assistant"<|sep|><|open|>think<|sep|>
\ No newline at end of file
rust/src/chat/src/renderer/kimi_k3/tests.rs
+13 -2
@@ -92,10 +92,15 @@ fn golden_dynamic_system_tool_declare() {
92 assert_golden("dynamic_system_tool_declare");
93 }
94
95 +#[test]
96 +fn golden_dynamic_only_tool_declare() {
97 + assert_golden("dynamic_only_tool_declare");
98 +}
99 +
100 #[test]
101 fn tool_declare_omits_absent_fields_and_preserves_parameter_nulls() {
102 let mut request = crate::request::ChatRequest::for_test();
98 - request.tools = vec![ChatTool {
103 + let tools = vec![ChatTool {
104 name: "lookup".to_string(),
105 description: None,
106 parameters: json!({
@@ -109,6 +114,9 @@ fn tool_declare_omits_absent_fields_and_preserves_parameter_nulls() {
114 }),
115 strict: None,
116 }];
117 + request.tool_context =
118 + crate::request::ResolvedToolContext::new(&request.messages, tools, None, true)
119 + .expect("tool context should resolve");
120
121 let rendered = render_request(&request);
122
@@ -122,12 +130,15 @@ fn tool_declare_omits_absent_fields_and_preserves_parameter_nulls() {
130 #[test]
131 fn tool_declare_preserves_present_optional_fields() {
132 let mut request = crate::request::ChatRequest::for_test();
125 - request.tools = vec![ChatTool {
133 + let tools = vec![ChatTool {
134 name: "lookup".to_string(),
135 description: Some("Look up a record".to_string()),
136 parameters: json!({"type": "object"}),
137 strict: Some(false),
138 }];
139 + request.tool_context =
140 + crate::request::ResolvedToolContext::new(&request.messages, tools, None, true)
141 + .expect("tool context should resolve");
142
143 let rendered = render_request(&request);
144
rust/src/chat/src/renderer/test_utils.rs
+12 -14
@@ -11,7 +11,7 @@ use serde_json::Value;
11 use crate::event::{AssistantContentBlock, AssistantToolCall};
12 use crate::request::{
13 ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice,
14 - GenerationPromptMode, ReasoningEffort,
14 + GenerationPromptMode, ReasoningEffort, ResolvedToolContext,
15 };
16
17 /// Options for constructing a [`ChatRequest`] from a fixture file.
@@ -158,22 +158,20 @@ struct FixtureToolCallFunction {
158 impl FixtureRequest {
159 fn into_chat_request(self, options: FixtureRequestOptions) -> ChatRequest {
160 let tools = to_chat_tools(&self.tools);
161 - let tool_choice = self.tool_choice.unwrap_or(if tools.is_empty() {
162 - ChatToolChoice::None
163 - } else {
164 - ChatToolChoice::Auto
165 - });
161 +
162 + let messages = self
163 + .messages
164 + .into_iter()
165 + .enumerate()
166 + .map(|(index, message)| fixture_message_to_chat_message(index, message))
167 + .collect::<Vec<_>>();
168 + let tool_context = ResolvedToolContext::new(&messages, tools, self.tool_choice, true)
169 + .expect("fixture tool context should resolve");
170
171 let mut request = ChatRequest {
172 request_id: "renderer-fixture".to_string(),
169 - messages: self
170 - .messages
171 - .into_iter()
172 - .enumerate()
173 - .map(|(index, message)| fixture_message_to_chat_message(index, message))
174 - .collect(),
175 - tools,
176 - tool_choice,
173 + messages,
174 + tool_context,
175 ..ChatRequest::for_test()
176 };
177
rust/src/chat/src/request.rs
+224 -14
@@ -1,7 +1,7 @@
1 // SPDX-License-Identifier: Apache-2.0
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -use std::collections::HashMap;
4 +use std::collections::{HashMap, HashSet};
5
6 use llm_multimodal::ImageDetail;
7 use serde::{Deserialize, Serialize};
@@ -248,6 +248,16 @@ impl ChatMessage {
248 }
249 }
250
251 + /// Return message-scoped tool declarations carried by this message.
252 + pub fn declared_tools(&self) -> Option<&[ChatTool]> {
253 + match self {
254 + Self::Developer {
255 + tools: Some(tools), ..
256 + } if !tools.is_empty() => Some(tools),
257 + _ => None,
258 + }
259 + }
260 +
261 /// Construct one chat message with user role.
262 pub fn user(content: impl Into<ChatContent>) -> Self {
263 Self::User {
@@ -450,6 +460,93 @@ pub enum ChatToolChoice {
460 },
461 }
462
463 +/// Resolved tool state shared by rendering, output parsing, and constraints.
464 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
465 +pub struct ResolvedToolContext {
466 + /// Request-level tools rendered before the ordered chat history.
467 + pub initial_tools: Vec<ChatTool>,
468 + /// Request-level and message-scoped tools available for this generation.
469 + pub effective_tools: Vec<ChatTool>,
470 + /// Resolved tool-choice behavior for this request.
471 + pub tool_choice: ChatToolChoice,
472 + /// Whether more than one parsed tool call may be surfaced northbound.
473 + pub parallel_tool_calls: bool,
474 +}
475 +
476 +impl ResolvedToolContext {
477 + /// Resolve one complete tool context from request and message declarations.
478 + pub fn new(
479 + messages: &[ChatMessage],
480 + initial_tools: Vec<ChatTool>,
481 + requested_tool_choice: Option<ChatToolChoice>,
482 + parallel_tool_calls: bool,
483 + ) -> Result<Self> {
484 + let dynamic_tool_count = messages
485 + .iter()
486 + .filter_map(ChatMessage::declared_tools)
487 + .map(<[ChatTool]>::len)
488 + .sum::<usize>();
489 + let mut effective_tools = Vec::with_capacity(initial_tools.len() + dynamic_tool_count);
490 + let mut names = HashSet::with_capacity(effective_tools.capacity());
491 +
492 + for tool in initial_tools
493 + .iter()
494 + .chain(messages.iter().filter_map(ChatMessage::declared_tools).flatten())
495 + {
496 + if !names.insert(tool.name.clone()) {
497 + return Err(Error::DuplicateToolName {
498 + name: tool.name.clone(),
499 + });
500 + }
501 + effective_tools.push(tool.clone());
502 + }
503 +
504 + let tool_choice = requested_tool_choice.unwrap_or({
505 + if effective_tools.is_empty() {
506 + ChatToolChoice::None
507 + } else {
508 + ChatToolChoice::Auto
509 + }
510 + });
511 +
512 + match &tool_choice {
513 + ChatToolChoice::Auto | ChatToolChoice::Required if effective_tools.is_empty() => {
514 + return Err(Error::ToolChoiceRequiresTools);
515 + }
516 + ChatToolChoice::Function { name } if !names.contains(name) => {
517 + return Err(Error::ToolChoiceFunctionNotFound { name: name.clone() });
518 + }
519 + ChatToolChoice::None
520 + | ChatToolChoice::Auto
521 + | ChatToolChoice::Required
522 + | ChatToolChoice::Function { .. } => {}
523 + }
524 +
525 + Ok(Self {
526 + initial_tools,
527 + effective_tools,
528 + tool_choice,
529 + parallel_tool_calls,
530 + })
531 + }
532 +
533 + /// Return whether tool output parsing is active for this request.
534 + pub fn parsing_enabled(&self) -> bool {
535 + !matches!(self.tool_choice, ChatToolChoice::None) && !self.effective_tools.is_empty()
536 + }
537 +}
538 +
539 +impl Default for ResolvedToolContext {
540 + fn default() -> Self {
541 + Self {
542 + initial_tools: Vec::new(),
543 + effective_tools: Vec::new(),
544 + tool_choice: ChatToolChoice::None,
545 + parallel_tool_calls: true,
546 + }
547 + }
548 +}
549 +
550 /// One chat request ready to be rendered into a prompt and lowered into a
551 /// generate request.
552 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
@@ -462,14 +559,8 @@ pub struct ChatRequest {
559 pub sampling_params: SamplingParams,
560 /// Chat-specific rendering options.
561 pub chat_options: ChatOptions,
465 - /// Function tools made available to the model for this request.
466 - pub tools: Vec<ChatTool>,
467 - /// Tool-choice behavior for this request.
468 - pub tool_choice: ChatToolChoice,
469 - /// Whether the model may return more than one tool call per response.
470 - ///
471 - /// When `false`, only the first parsed tool call is surfaced northbound.
472 - pub parallel_tool_calls: bool,
562 + /// Resolved request-level and message-scoped tool behavior.
563 + pub tool_context: ResolvedToolContext,
564 /// Text decode options for incremental detokenization.
565 pub decode_options: TextDecodeOptions,
566 /// Whether to emit intermediate northbound content deltas before the
@@ -507,9 +598,7 @@ impl ChatRequest {
598 messages: vec![ChatMessage::text(ChatRole::User, "test")],
599 sampling_params: SamplingParams::default(),
600 chat_options: ChatOptions::default(),
510 - tools: Vec::new(),
511 - tool_choice: ChatToolChoice::None,
512 - parallel_tool_calls: true,
601 + tool_context: ResolvedToolContext::default(),
602 decode_options: TextDecodeOptions::default(),
603 intermediate: true,
604 priority: 0,
@@ -547,10 +636,33 @@ impl ChatRequest {
636 self.messages.iter().any(ChatMessage::has_multimodal)
637 }
638
639 + /// Return every tool available for this generation.
640 + pub fn tools(&self) -> &[ChatTool] {
641 + &self.tool_context.effective_tools
642 + }
643 +
644 + /// Return request-level tools before message-scoped declarations.
645 + ///
646 + /// Renderers should use [`Self::tools`] unless their prompt format places
647 + /// initial and message-scoped declarations differently.
648 + pub fn initial_tools(&self) -> &[ChatTool] {
649 + &self.tool_context.initial_tools
650 + }
651 +
652 + /// Return the resolved tool-choice behavior.
653 + pub fn tool_choice(&self) -> &ChatToolChoice {
654 + &self.tool_context.tool_choice
655 + }
656 +
657 + /// Return whether more than one parsed tool call may be surfaced.
658 + pub fn parallel_tool_calls(&self) -> bool {
659 + self.tool_context.parallel_tool_calls
660 + }
661 +
662 /// Return true if this request should enable tool parsing based on the tool
663 /// choice and tool list.
664 pub(crate) fn tool_parsing_enabled(&self) -> bool {
553 - !matches!(self.tool_choice, ChatToolChoice::None) && !self.tools.is_empty()
665 + self.tool_context.parsing_enabled()
666 }
667
668 /// Return the request-level thinking toggle when explicitly requested.
@@ -605,7 +717,10 @@ impl ChatRole {
717 mod tests {
718 use serde_json::{json, to_value};
719
608 - use super::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool};
720 + use super::{
721 + ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice,
722 + ResolvedToolContext,
723 + };
724 use crate::Error;
725 use crate::event::AssistantContentBlock;
726
@@ -702,6 +817,101 @@ mod tests {
817 assert_eq!(decoded, message);
818 }
819
820 + fn chat_tool(name: &str) -> ChatTool {
821 + ChatTool {
822 + name: name.to_string(),
823 + description: None,
824 + parameters: json!({"type": "object"}),
825 + strict: None,
826 + }
827 + }
828 +
829 + #[test]
830 + fn resolved_tool_context_defaults_dynamic_only_tools_to_auto() {
831 + let messages = vec![
832 + ChatMessage::user("hello"),
833 + ChatMessage::developer("", Some(vec![chat_tool("lookup")])),
834 + ];
835 +
836 + let context = ResolvedToolContext::new(&messages, Vec::new(), None, false).unwrap();
837 +
838 + assert!(context.initial_tools.is_empty());
839 + assert_eq!(context.effective_tools, vec![chat_tool("lookup")]);
840 + assert_eq!(context.tool_choice, ChatToolChoice::Auto);
841 + assert!(!context.parallel_tool_calls);
842 + assert!(context.parsing_enabled());
843 + }
844 +
845 + #[test]
846 + fn resolved_tool_context_preserves_initial_then_message_order() {
847 + let messages = vec![
848 + ChatMessage::developer("", Some(vec![chat_tool("middle")])),
849 + ChatMessage::developer("policy", Some(vec![chat_tool("last")])),
850 + ];
851 +
852 + let context = ResolvedToolContext::new(
853 + &messages,
854 + vec![chat_tool("first")],
855 + Some(ChatToolChoice::Required),
856 + true,
857 + )
858 + .unwrap();
859 +
860 + assert_eq!(
861 + context
862 + .effective_tools
863 + .iter()
864 + .map(|tool| tool.name.as_str())
865 + .collect::<Vec<_>>(),
866 + vec!["first", "middle", "last"]
867 + );
868 + assert_eq!(context.tool_choice, ChatToolChoice::Required);
869 + }
870 +
871 + #[test]
872 + fn resolved_tool_context_rejects_duplicate_names() {
873 + let messages = vec![ChatMessage::developer("", Some(vec![chat_tool("lookup")]))];
874 +
875 + let error =
876 + ResolvedToolContext::new(&messages, vec![chat_tool("lookup")], None, true).unwrap_err();
877 +
878 + assert!(matches!(error, Error::DuplicateToolName { name } if name == "lookup"));
879 + }
880 +
881 + #[test]
882 + fn resolved_tool_context_validates_named_choice_against_effective_tools() {
883 + let messages = vec![ChatMessage::developer("", Some(vec![chat_tool("lookup")]))];
884 + let context = ResolvedToolContext::new(
885 + &messages,
886 + Vec::new(),
887 + Some(ChatToolChoice::Function {
888 + name: "lookup".to_string(),
889 + }),
890 + true,
891 + )
892 + .unwrap();
893 + assert_eq!(
894 + context.tool_choice,
895 + ChatToolChoice::Function {
896 + name: "lookup".to_string()
897 + }
898 + );
899 +
900 + let error = ResolvedToolContext::new(
901 + &messages,
902 + Vec::new(),
903 + Some(ChatToolChoice::Function {
904 + name: "missing".to_string(),
905 + }),
906 + true,
907 + )
908 + .unwrap_err();
909 + assert!(matches!(
910 + error,
911 + Error::ToolChoiceFunctionNotFound { name } if name == "missing"
912 + ));
913 + }
914 +
915 #[test]
916 fn enable_thinking_is_none_when_no_kwargs_are_present() {
917 let request = ChatRequest::for_test();
rust/src/chat/tests/chat.rs
+8 -2
@@ -302,7 +302,7 @@ fn sample_request(request_id: &str) -> ChatRequest {
302
303 fn sample_tool_request(request_id: &str) -> ChatRequest {
304 let mut request = sample_request(request_id);
305 - request.tools = vec![ChatTool {
305 + let tools = vec![ChatTool {
306 name: "get_weather".to_string(),
307 description: Some("Get weather".to_string()),
308 parameters: serde_json::json!({
@@ -312,7 +312,13 @@ fn sample_tool_request(request_id: &str) -> ChatRequest {
312 }),
313 strict: None,
314 }];
315 - request.tool_choice = ChatToolChoice::Auto;
315 + request.tool_context = vllm_chat::ResolvedToolContext::new(
316 + &request.messages,
317 + tools,
318 + Some(ChatToolChoice::Auto),
319 + true,
320 + )
321 + .expect("tool context should resolve");
322 request
323 }
324
rust/src/chat/tests/roundtrip.rs
+6 -9
@@ -16,9 +16,9 @@ use serde_json_fmt::JsonFormat as JsonFmt;
16 use serial_test::file_serial;
17 use vllm_chat::{
18 AssistantContentBlock, AssistantMessage, AssistantMessageExt as _, AssistantToolCall,
19 - ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice, FinishReason,
20 - GenerationPromptMode, LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection,
21 - RendererSelection, load_model_backends,
19 + ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, FinishReason, GenerationPromptMode,
20 + LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, RendererSelection,
21 + load_model_backends,
22 };
23 use vllm_text::{DecodedTextEvent, Finished, Prompt};
24 use vllm_tokenizer::Tokenizer;
@@ -825,15 +825,12 @@ fn roundtrip_request(
825 tools: Vec<ChatTool>,
826 thinking: Option<bool>,
827 ) -> ChatRequest {
828 + let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, None, true)
829 + .expect("tool context should resolve");
830 let mut request = ChatRequest {
831 request_id: request_id.into(),
832 messages,
831 - tool_choice: if tools.is_empty() {
832 - ChatToolChoice::None
833 - } else {
834 - ChatToolChoice::Auto
835 - },
836 - tools,
833 + tool_context,
834 ..ChatRequest::for_test()
835 };
836
rust/src/server/src/routes/openai/chat_completions/convert.rs
+38 -25
@@ -5,16 +5,16 @@ use itertools::Itertools as _;
5 use vllm_chat::{
6 AssistantContentBlock, AssistantToolCall, ChatContent, ChatContentPart,
7 ChatMessage as VllmChatMessage, ChatOptions, ChatRequest, ChatTool, ChatToolChoice,
8 - GenerationPromptMode, SamplingParams,
8 + GenerationPromptMode, ResolvedToolContext, SamplingParams,
9 };
10
11 use super::types::ChatCompletionRequest;
12 use super::validate;
13 -use crate::error::{ApiError, bail_invalid_request};
13 +use crate::error::{ApiError, bail_invalid_request, chat_submit_error};
14 use crate::lora::LoraModelResolution;
15 use crate::routes::openai::utils::structured_outputs::convert_from_response_format;
16 use crate::routes::openai::utils::types::{
17 - ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue,
17 + ChatMessage, ContentPart, MessageContent, Tool, ToolChoice, ToolChoiceValue, ToolReference,
18 };
19 use crate::utils::{
20 ResolvedRequestContext, convert_logit_bias, merge_ec_transfer_params, merge_kv_transfer_params,
@@ -130,6 +130,14 @@ pub(super) fn prepare_chat_request(
130 request.vllm_xargs.as_ref(),
131 );
132
133 + let tool_context = ResolvedToolContext::new(
134 + &messages,
135 + convert_tools(request.tools)?,
136 + request.tool_choice.as_ref().map(convert_tool_choice).transpose()?,
137 + request.parallel_tool_calls.unwrap_or(true),
138 + )
139 + .map_err(|error| chat_submit_error("failed to resolve request tools", error))?;
140 +
141 let chat_request = ChatRequest {
142 request_id: request_id.clone(),
143 messages,
@@ -168,9 +176,7 @@ pub(super) fn prepare_chat_request(
176 response_format,
177 template_kwargs,
178 },
171 - tools: convert_tools(request.tools)?,
172 - tool_choice: convert_tool_choice(request.tool_choice.as_ref())?,
173 - parallel_tool_calls: request.parallel_tool_calls.unwrap_or(true),
179 + tool_context,
180 decode_options: vllm_text::output::TextDecodeOptions {
181 skip_special_tokens: request.skip_special_tokens,
182 include_stop_str_in_output: request.include_stop_str_in_output,
@@ -399,17 +405,21 @@ fn convert_message_tools(tools: Option<Vec<Tool>>) -> Result<Option<Vec<ChatTool
405 Ok((!tools.is_empty()).then_some(tools))
406 }
407
402 -fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result<ChatToolChoice, ApiError> {
408 +fn convert_tool_choice(tool_choice: &ToolChoice) -> Result<ChatToolChoice, ApiError> {
409 match tool_choice {
404 - None | Some(ToolChoice::Value(ToolChoiceValue::Auto)) => Ok(ChatToolChoice::Auto),
405 - Some(ToolChoice::Value(ToolChoiceValue::None)) => Ok(ChatToolChoice::None),
406 - Some(ToolChoice::Value(ToolChoiceValue::Required)) => Ok(ChatToolChoice::Required),
407 - Some(ToolChoice::Function {
410 + ToolChoice::Value(ToolChoiceValue::Auto) => Ok(ChatToolChoice::Auto),
411 + ToolChoice::Value(ToolChoiceValue::None) => Ok(ChatToolChoice::None),
412 + ToolChoice::Value(ToolChoiceValue::Required) => Ok(ChatToolChoice::Required),
413 + ToolChoice::Function {
414 tool_type,
415 function,
410 - }) if tool_type == "function" => Ok(ChatToolChoice::Function {
416 + } if tool_type == "function" => Ok(ChatToolChoice::Function {
417 name: function.name.clone(),
418 }),
419 + ToolChoice::AllowedTools { tools, .. } => bail_invalid_request!(
420 + "allowed_tools tool_choice is not supported yet: {}.",
421 + tools.iter().map(ToolReference::identifier).join(", ")
422 + ),
423 _ => bail_invalid_request!("tool_choice={:?} is not supported yet.", tool_choice),
424 }
425 }
@@ -478,7 +488,7 @@ mod tests {
488 )
489 .expect("request is valid");
490
481 - assert!(!prepared.chat_request.parallel_tool_calls);
491 + assert!(!prepared.chat_request.parallel_tool_calls());
492 }
493
494 #[test]
@@ -490,7 +500,7 @@ mod tests {
500 )
501 .expect("request is valid");
502
493 - assert!(prepared.chat_request.parallel_tool_calls);
503 + assert!(prepared.chat_request.parallel_tool_calls());
504 }
505
506 #[test]
@@ -596,8 +606,8 @@ mod tests {
606 min_tokens: 0,
607 }
608 );
599 - assert!(prepared.chat_request.tools.is_empty());
600 - assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
609 + assert!(prepared.chat_request.initial_tools().is_empty());
610 + assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
611 }
612
613 #[test]
@@ -671,8 +681,8 @@ mod tests {
681 min_tokens: 0,
682 }
683 );
674 - assert!(prepared.chat_request.tools.is_empty());
675 - assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
684 + assert!(prepared.chat_request.initial_tools().is_empty());
685 + assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
686 }
687
688 #[test]
@@ -1009,8 +1019,8 @@ mod tests {
1019 },
1020 ])]
1021 );
1012 - assert!(prepared.chat_request.tools.is_empty());
1013 - assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Auto);
1022 + assert!(prepared.chat_request.initial_tools().is_empty());
1023 + assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
1024 }
1025
1026 #[test]
@@ -1105,7 +1115,7 @@ mod tests {
1115 ]
1116 );
1117 assert_eq!(
1108 - prepared.chat_request.tools,
1118 + prepared.chat_request.initial_tools(),
1119 vec![VllmChatTool {
1120 name: "get_weather".to_string(),
1121 description: Some("Get weather".to_string()),
@@ -1116,7 +1126,7 @@ mod tests {
1126 strict: None,
1127 }]
1128 );
1119 - assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::None);
1129 + assert_eq!(prepared.chat_request.tool_choice(), &ChatToolChoice::None);
1130 }
1131
1132 #[test]
@@ -1145,7 +1155,10 @@ mod tests {
1155 )
1156 .expect("request is valid");
1157
1148 - assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Required);
1158 + assert_eq!(
1159 + prepared.chat_request.tool_choice(),
1160 + &ChatToolChoice::Required
1161 + );
1162 assert!(!prepared.options.is_named_tool_choice);
1163 }
1164
@@ -1181,8 +1194,8 @@ mod tests {
1194 .expect("request is valid");
1195
1196 assert_eq!(
1184 - prepared.chat_request.tool_choice,
1185 - ChatToolChoice::Function {
1197 + prepared.chat_request.tool_choice(),
1198 + &ChatToolChoice::Function {
1199 name: "get_weather".to_string(),
1200 }
1201 );
rust/src/server/src/routes/openai/chat_completions/types.rs
+2 -109
@@ -14,8 +14,8 @@ use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
14 use crate::routes::openai::utils::structured_outputs::ResponseFormat;
15 use crate::routes::openai::utils::types::{
16 ChatLogProbs, ChatMessage, Normalizable, StreamOptions, StringOrArray, Tool, ToolCall,
17 - ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, UNKNOWN_MODEL_ID, Usage,
18 - default_true, validate_messages, validate_stop, validate_top_p_value,
17 + ToolCallDelta, ToolChoice, UNKNOWN_MODEL_ID, Usage, default_true, validate_messages,
18 + validate_stop, validate_top_p_value,
19 };
20
21 /// vLLM-compatible request type for the Chat Completions API.
@@ -325,19 +325,6 @@ impl Normalizable for ChatCompletionRequest {
325 self.max_completion_tokens = self.max_tokens;
326 self.max_tokens = None;
327 }
328 -
329 - // Apply tool_choice defaults
330 - // If tools is None, leave tool_choice as None (don't set it)
331 - if self.tool_choice.is_none()
332 - && let Some(tools) = &self.tools
333 - {
334 - let choice_value = if tools.is_empty() {
335 - ToolChoiceValue::None
336 - } else {
337 - ToolChoiceValue::Auto
338 - };
339 - self.tool_choice = Some(ToolChoice::Value(choice_value));
340 - }
328 }
329 }
330
@@ -492,99 +479,5 @@ fn validate_chat_cross_parameters(
479 return Err(e);
480 }
481
495 - // 5. Validate tool_choice requires tools (except for "none")
496 - if let Some(ref tool_choice) = req.tool_choice {
497 - let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
498 -
499 - // Check if tool_choice is anything other than "none"
500 - let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None));
501 -
502 - if is_some_choice && !has_tools {
503 - let mut e = validator::ValidationError::new("tool_choice_requires_tools");
504 - e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into());
505 - return Err(e);
506 - }
507 -
508 - // Additional validation when tools are present
509 - if let Some(tools) = req.tools.as_ref().filter(|t| !t.is_empty()) {
510 - match tool_choice {
511 - ToolChoice::Function { function, .. } => {
512 - // Validate that the specified function name exists in tools
513 - let function_exists = tools.iter().any(|tool| {
514 - tool.tool_type == "function" && tool.function.name == function.name
515 - });
516 -
517 - if !function_exists {
518 - let mut e =
519 - validator::ValidationError::new("tool_choice_function_not_found");
520 - e.message = Some(
521 - format!(
522 - "Invalid value for 'tool_choice': function '{}' not found in 'tools'.",
523 - function.name
524 - )
525 - .into(),
526 - );
527 - return Err(e);
528 - }
529 - }
530 - ToolChoice::AllowedTools {
531 - mode,
532 - tools: allowed_tools,
533 - ..
534 - } => {
535 - // Validate mode is "auto" or "required"
536 - if mode != "auto" && mode != "required" {
537 - let mut e = validator::ValidationError::new("tool_choice_invalid_mode");
538 - e.message = Some(format!(
539 - "Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{mode}'."
540 - ).into());
541 - return Err(e);
542 - }
543 -
544 - // Validate that all ToolReferences are Function type (Chat API only supports
545 - // function tools)
546 - for tool_ref in allowed_tools {
547 - match tool_ref {
548 - ToolReference::Function { name } => {
549 - // Validate that the function exists in tools array
550 - let tool_exists = tools.iter().any(|tool| {
551 - tool.tool_type == "function" && tool.function.name == *name
552 - });
553 -
554 - if !tool_exists {
555 - let mut e = validator::ValidationError::new(
556 - "tool_choice_tool_not_found",
557 - );
558 - e.message = Some(
559 - format!(
560 - "Invalid value for 'tool_choice.tools': tool '{name}' not found in 'tools'."
561 - )
562 - .into(),
563 - );
564 - return Err(e);
565 - }
566 - }
567 - _ => {
568 - // Chat Completion API only supports function tools in tool_choice
569 - let mut e = validator::ValidationError::new(
570 - "tool_choice_invalid_tool_type",
571 - );
572 - e.message = Some(
573 - format!(
574 - "Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.",
575 - tool_ref.identifier()
576 - )
577 - .into(),
578 - );
579 - return Err(e);
580 - }
581 - }
582 - }
583 - }
584 - ToolChoice::Value(_) => {}
585 - }
586 - }
587 - }
588 -
482 Ok(())
483 }
rust/src/server/src/routes/tokenize/types.rs
+8 -5
@@ -7,7 +7,7 @@ use itertools::Itertools as _;
7 use serde::{Deserialize, Serialize};
8 use serde_json::Value;
9 use validator::{Validate, ValidationErrors};
10 -use vllm_chat::{ChatOptions, ChatRequest, ChatToolChoice, SamplingParams};
10 +use vllm_chat::{ChatOptions, ChatRequest, ResolvedToolContext, SamplingParams};
11 use vllm_text::output::TextDecodeOptions;
12 use vllm_text::{Prompt, TextRequest};
13
@@ -97,6 +97,11 @@ impl TokenizeChatRequest {
97 &messages,
98 )?;
99
100 + let tool_context =
101 + ResolvedToolContext::new(&messages, convert_tools(self.tools)?, None, true).map_err(
102 + |error| crate::error::chat_submit_error("failed to resolve request tools", error),
103 + )?;
104 +
105 Ok(ChatRequest {
106 request_id,
107 messages,
@@ -108,9 +113,7 @@ impl TokenizeChatRequest {
113 response_format: None,
114 template_kwargs: self.chat_template_kwargs.unwrap_or_default(),
115 },
111 - tools: convert_tools(self.tools)?,
112 - tool_choice: ChatToolChoice::Auto,
113 - parallel_tool_calls: true,
116 + tool_context,
117 decode_options: TextDecodeOptions::default(),
118 intermediate: false,
119 priority: 0,
@@ -199,7 +202,7 @@ mod tests {
202 req.into_chat_request("tokenize-test".to_string()).expect("request is valid");
203
204 assert_eq!(
202 - chat_request.tools,
205 + chat_request.initial_tools(),
206 vec![ChatTool {
207 name: "get_weather".to_string(),
208 description: Some("Get weather".to_string()),