@tduncan / Nvidia-Optimization-Stack-VLLM / commits / e52da038d1

[Rust Frontend] Normalize native renderer reasoning controls (#56998)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Sep 17, 2026 at 09:15 UTC e52da038d19c202ede695a4273adc2316b4ba1a7
29 files changed +1252 -453
rust/src/chat/src/backend/hf.rs
+53 -6
@@ -70,12 +70,26 @@ impl HfChatBackend {
70 options,
71 multimodal_render_info,
72 )?),
73 - RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new()),
74 - RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new()),
75 - RendererSelection::DeepSeekV41 => Arc::new(DeepSeekV41ChatRenderer::new()),
76 - RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new()?),
77 - RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(tokenizer.clone())?),
78 - RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(tokenizer.clone())),
73 + RendererSelection::DeepSeekV32 => Arc::new(DeepSeekV32ChatRenderer::new(
74 + options.default_chat_template_kwargs,
75 + )),
76 + RendererSelection::DeepSeekV4 => Arc::new(DeepSeekV4ChatRenderer::new(
77 + options.default_chat_template_kwargs,
78 + )),
79 + RendererSelection::DeepSeekV41 => Arc::new(DeepSeekV41ChatRenderer::new(
80 + options.default_chat_template_kwargs,
81 + )),
82 + RendererSelection::Harmony => Arc::new(HarmonyChatRenderer::new(
83 + options.default_chat_template_kwargs,
84 + )?),
85 + RendererSelection::Inkling => Arc::new(InklingChatRenderer::new(
86 + tokenizer.clone(),
87 + options.default_chat_template_kwargs,
88 + )?),
89 + RendererSelection::KimiK3 => Arc::new(KimiK3ChatRenderer::new(
90 + tokenizer.clone(),
91 + options.default_chat_template_kwargs,
92 + )),
93 };
94
95 info!(
@@ -280,6 +294,39 @@ mod tests {
294 assert_eq!(prompt, "hello");
295 }
296
297 + #[test]
298 + fn native_renderer_inherits_deployment_reasoning_below_request_controls() {
299 + let backend = HfChatBackend::from_resolved_model_files(
300 + resolved_files(r#"{"model_type":"deepseek_v4"}"#, "{}"),
301 + "test-model".to_string(),
302 + LoadModelBackendsOptions {
303 + default_chat_template_kwargs: [
304 + ("thinking".to_string(), serde_json::json!(false)),
305 + ("reasoning_effort".to_string(), serde_json::json!("low")),
306 + ]
307 + .into(),
308 + ..Default::default()
309 + },
310 + test_tokenizer(),
311 + )
312 + .unwrap();
313 + let mut request = request_with_user_text("hello");
314 + let rendered = backend.chat_renderer().render(&request).unwrap();
315 + assert!(rendered.prompt.into_text().unwrap().ends_with("</think>"));
316 + assert_eq!(rendered.effective_template_kwargs["enable_thinking"], false);
317 +
318 + request.chat_options.reasoning_effort = Some(crate::EffortValue::from("max"));
319 + let rendered = backend.chat_renderer().render(&request).unwrap();
320 + let prompt = rendered.prompt.into_text().unwrap();
321 + assert!(prompt.contains("Reasoning Effort: Beyond maximum"));
322 + assert!(prompt.ends_with("<think>"));
323 + assert_eq!(
324 + rendered.effective_template_kwargs["reasoning_effort"],
325 + "max"
326 + );
327 + assert_eq!(rendered.effective_template_kwargs["thinking"], true);
328 + }
329 +
330 #[test]
331 fn auto_uses_harmony_renderer_and_output_processor_for_gpt_oss_model_type() {
332 let backend = backend_for_selection(
rust/src/chat/src/backend/mod.rs
+2 -2
@@ -78,8 +78,8 @@ pub struct LoadModelBackendsOptions {
78 /// Optional server-default chat template override, provided either as an
79 /// inline template or as a path to a template file.
80 pub chat_template: Option<String>,
81 - /// Optional server-default keyword arguments merged into every
82 - /// chat-template render before request-level `chat_template_kwargs`.
81 + /// Server-default keyword arguments. HF merges these before request kwargs;
82 + /// native renderers inherit supported reasoning controls below request controls.
83 pub default_chat_template_kwargs: HashMap<String, Value>,
84 /// Maximum number of input items allowed per prompt for each modality.
85 /// Unspecified modalities are unlimited.
rust/src/chat/src/error.rs
+3
@@ -19,6 +19,8 @@ pub enum Error {
19 ChatTemplate(String),
20 #[error("{0}")]
21 InvalidReasoningEffort(String),
22 + #[error("{message}")]
23 + InvalidReasoningControl { message: String },
24 #[error("multimodal input is not supported by this chat renderer")]
25 UnsupportedMultimodalRenderer,
26 #[error("unsupported multimodal content: {0}")]
@@ -101,6 +103,7 @@ impl Error {
103 match self {
104 Self::PromptTooLong { .. }
105 | Self::InvalidReasoningEffort(_)
106 + | Self::InvalidReasoningControl { .. }
107 | Self::DuplicateToolName { .. }
108 | Self::ToolChoiceRequiresTools
109 | Self::ToolChoiceFunctionNotFound { .. } => true,
rust/src/chat/src/lib.rs
+3 -1
@@ -31,6 +31,7 @@ pub use parser::reasoning::{
31 };
32 pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory};
33 pub use parser::{ParserSelection, validate_parser_overrides};
34 +pub use reasoning::EffortValue;
35 pub use renderer::hf::ChatTemplateContentFormatOption;
36 pub use renderer::{
37 ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer,
@@ -39,7 +40,7 @@ pub use renderer::{
40 };
41 pub use request::{
42 ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
42 - ChatToolChoice, GenerationPromptMode, ReasoningEffort, ResolvedToolContext, SamplingParams,
43 + ChatToolChoice, GenerationPromptMode, ResolvedToolContext, SamplingParams,
44 };
45 pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
46 pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
@@ -52,6 +53,7 @@ mod event;
53 pub mod multimodal;
54 mod output;
55 mod parser;
56 +mod reasoning;
57 mod renderer;
58 mod request;
59 mod stream;
rust/src/chat/src/reasoning.rs new
+397
@@ -0,0 +1,397 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +//! Shared reasoning-control resolution for native renderers.
5 +//!
6 +//! Each input source is parsed independently into [`ReasoningControl`]. Sources
7 +//! then fall back in priority order: request, deployment defaults, model default.
8 +//! Within a source, an explicit toggle decides the mode and effort supplies its
9 +//! intensity. Keeping sources separate lets request effort override a deployment
10 +//! toggle, and lets an enabled request inherit effort from enabled defaults.
11 +//!
12 +//! This module owns input types, precedence, and the standard `"none"` sentinel.
13 +//! Model adapters own supported effort names, numeric ranges, and prompt encoding.
14 +//! After model lowering, [`ReasoningControl::template_kwargs`] projects the prompt's
15 +//! decision for downstream consumers. HF prepares raw template inputs separately.
16 +
17 +use std::collections::HashMap;
18 +use std::fmt;
19 +
20 +use serde::{Deserialize, Serialize};
21 +use serde_json::{Number, Value, json};
22 +
23 +use crate::error::{Result, invalid_reasoning_control};
24 +use crate::request::ChatRequest;
25 +
26 +/// A model-specific reasoning effort, before or after renderer lowering.
27 +///
28 +/// Strings preserve model-specific names; JSON numbers preserve integer and
29 +/// floating-point representations for model-local range validation. Missing/null
30 +/// effort is represented by `Option::None` in [`crate::ChatOptions`].
31 +/// Supported names and ranges are validated by each renderer or HF template.
32 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
33 +#[serde(untagged)]
34 +pub enum EffortValue {
35 + String(String),
36 + Number(Number),
37 +}
38 +
39 +impl TryFrom<Value> for EffortValue {
40 + type Error = crate::error::Error;
41 +
42 + fn try_from(value: Value) -> Result<Self> {
43 + match value {
44 + Value::String(value) => Ok(Self::String(value)),
45 + Value::Number(value) => Ok(Self::Number(value)),
46 + _ => Err(invalid_reasoning_control!(
47 + "reasoning effort must be a string or number, got {value}"
48 + )),
49 + }
50 + }
51 +}
52 +
53 +impl TryFrom<f64> for EffortValue {
54 + type Error = crate::error::Error;
55 +
56 + /// Construct a numeric effort, rejecting NaN and infinities.
57 + fn try_from(value: f64) -> Result<Self> {
58 + Number::from_f64(value).map(Self::Number).ok_or_else(|| {
59 + invalid_reasoning_control!("reasoning effort must be finite, got {value}")
60 + })
61 + }
62 +}
63 +
64 +impl From<&str> for EffortValue {
65 + fn from(value: &str) -> Self {
66 + Self::String(value.to_owned())
67 + }
68 +}
69 +
70 +impl fmt::Display for EffortValue {
71 + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
72 + match self {
73 + Self::String(value) => json!(value).fmt(f),
74 + Self::Number(value) => value.fmt(f),
75 + }
76 + }
77 +}
78 +
79 +impl EffortValue {
80 + pub fn as_str(&self) -> Option<&str> {
81 + match self {
82 + Self::String(value) => Some(value),
83 + Self::Number(_) => None,
84 + }
85 + }
86 +
87 + pub fn as_f64(&self) -> Option<f64> {
88 + match self {
89 + Self::Number(value) => value.as_f64(),
90 + Self::String(_) => None,
91 + }
92 + }
93 +}
94 +
95 +/// Reasoning intent for one source, or the result of falling back across sources.
96 +#[derive(Debug, Clone, PartialEq)]
97 +pub(crate) enum ReasoningControl {
98 + /// This source leaves the mode and effort to a lower-priority source.
99 + Default,
100 + /// Reasoning is explicitly disabled; lower-priority defaults preserve this mode.
101 + Disabled,
102 + /// Reasoning is enabled. Missing effort inherits from an enabled default.
103 + Enabled { effort: Option<EffortValue> },
104 +}
105 +
106 +impl ReasoningControl {
107 + /// Parse request controls; the typed effort field takes precedence over kwargs effort.
108 + pub(crate) fn from_request(request: &ChatRequest) -> Result<Self> {
109 + Self::from_kwargs(
110 + &request.chat_options.template_kwargs,
111 + request.chat_options.reasoning_effort.as_ref(),
112 + )
113 + }
114 +
115 + /// Parse a deployment source, whose controls all come from template kwargs.
116 + pub(crate) fn from_template_kwargs(kwargs: &HashMap<String, Value>) -> Result<Self> {
117 + Self::from_kwargs(kwargs, None)
118 + }
119 +
120 + /// Select controls within one source, then derive its mode and optional effort.
121 + fn from_kwargs(
122 + kwargs: &HashMap<String, Value>,
123 + typed_effort: Option<&EffortValue>,
124 + ) -> Result<Self> {
125 + // Presence sets alias priority: `thinking` wins, including when its value
126 + // is invalid. Validate the selected spelling as a boolean.
127 + let toggle = ["thinking", "enable_thinking"]
128 + .into_iter()
129 + .find_map(|key| kwargs.get(key).map(|value| (key, value)))
130 + .map(|(key, value)| {
131 + value.as_bool().ok_or_else(|| {
132 + invalid_reasoning_control!(
133 + "template kwarg `{key}` must be a boolean, got {value}"
134 + )
135 + })
136 + })
137 + .transpose()?;
138 +
139 + // The typed request field wins over kwargs. Parse kwargs only when used;
140 + // disabled mode discards effort, and JSON null means omitted effort.
141 + let effort = match (toggle, typed_effort) {
142 + (Some(false), _) => None,
143 + (_, Some(effort)) => Some(effort.clone()),
144 + (_, None) => kwargs
145 + .get("reasoning_effort")
146 + .filter(|effort| !effort.is_null())
147 + .cloned()
148 + .map(EffortValue::try_from)
149 + .transpose()?,
150 + };
151 +
152 + match (toggle, effort) {
153 + // 1. Explicit false disables reasoning and discards any effort,
154 + // including values that would fail validation in an enabled mode.
155 + (Some(false), _) => Ok(Self::Disabled),
156 + // 2. `none` disables reasoning when the toggle is absent. Explicit true
157 + // enables reasoning and leaves effort to enabled deployment/model defaults.
158 + (toggle, Some(EffortValue::String(effort))) if effort == "none" => {
159 + Ok(if toggle == Some(true) {
160 + Self::Enabled { effort: None }
161 + } else {
162 + Self::Disabled
163 + })
164 + }
165 + // 3. Absent toggle and absent/null effort leave the entire decision
166 + // to the next source in the fallback chain.
167 + (None, None) => Ok(Self::Default),
168 + // 4. Explicit true or a concrete effort enables reasoning;
169 + // missing effort inherits later.
170 + (_, effort) => Ok(Self::Enabled { effort }),
171 + }
172 + }
173 +
174 + /// Parse request and deployment independently, then apply request precedence.
175 + ///
176 + /// Each source validates its selected input types. Model adapters subsequently
177 + /// apply their model default and validate/map the resulting effective effort.
178 + pub(crate) fn resolve(
179 + request: &ChatRequest,
180 + defaults: &HashMap<String, Value>,
181 + ) -> Result<Self> {
182 + Ok(Self::from_request(request)?.fallback(Self::from_template_kwargs(defaults)?))
183 + }
184 +
185 + /// Fill this higher-priority source's missing decision or effort from `defaults`.
186 + ///
187 + /// `Default` inherits the whole lower-priority state. `Enabled { effort: None }`
188 + /// inherits effort only from enabled defaults and retains its enabled mode.
189 + /// Explicit disabling and concrete effort are final at this priority level.
190 + pub(crate) fn fallback(self, defaults: Self) -> Self {
191 + match (self, defaults) {
192 + (Self::Default, defaults) => defaults,
193 + (Self::Enabled { effort: None }, Self::Enabled { effort }) => Self::Enabled { effort },
194 + (request, _) => request,
195 + }
196 + }
197 +
198 + /// Construct an enabled model default or a model-lowered concrete effort.
199 + pub(crate) fn enabled(effort: impl Into<EffortValue>) -> Self {
200 + Self::Enabled {
201 + effort: Some(effort.into()),
202 + }
203 + }
204 +
205 + /// Report the resolved mode; callers apply model defaults before rendering.
206 + pub(crate) fn is_enabled(&self) -> bool {
207 + matches!(self, Self::Enabled { .. })
208 + }
209 +
210 + /// Return the active effort, which may still be missing before model fallback.
211 + pub(crate) fn effort(&self) -> Option<&EffortValue> {
212 + match self {
213 + Self::Enabled { effort } => effort.as_ref(),
214 + Self::Default | Self::Disabled => None,
215 + }
216 + }
217 +
218 + /// Project the same model-lowered decision used to render the native prompt.
219 + ///
220 + /// Preserve request extras and write both toggle aliases plus the effective
221 + /// effort. Disabled mode emits `"none"`; enabled mode with no effective effort
222 + /// clears the original effort key. `Default` preserves request kwargs.
223 + pub(crate) fn template_kwargs(&self, request: &ChatRequest) -> HashMap<String, Value> {
224 + let mut kwargs = request.chat_options.template_kwargs.clone();
225 + if matches!(self, Self::Default) {
226 + return kwargs;
227 + }
228 + kwargs.insert("thinking".to_string(), json!(self.is_enabled()));
229 + kwargs.insert("enable_thinking".to_string(), json!(self.is_enabled()));
230 + match self {
231 + Self::Disabled => {
232 + kwargs.insert("reasoning_effort".to_string(), json!("none"));
233 + }
234 + Self::Enabled {
235 + effort: Some(effort),
236 + } => {
237 + kwargs.insert("reasoning_effort".to_string(), json!(effort));
238 + }
239 + Self::Enabled { effort: None } => {
240 + kwargs.remove("reasoning_effort");
241 + }
242 + Self::Default => unreachable!(),
243 + }
244 + kwargs
245 + }
246 +}
247 +
248 +#[cfg(test)]
249 +mod tests {
250 + use super::*;
251 +
252 + fn kwargs(value: Value) -> HashMap<String, Value> {
253 + serde_json::from_value(value).unwrap()
254 + }
255 +
256 + #[test]
257 + fn numeric_effort_rejects_non_finite_values() {
258 + for value in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
259 + assert!(EffortValue::try_from(value).unwrap_err().is_request_validation_error());
260 + }
261 + }
262 +
263 + #[test]
264 + fn explicit_toggle_decides_mode_and_none_inherits_enabled_effort() {
265 + use ReasoningControl::{Default, Disabled, Enabled};
266 +
267 + for (source, expected) in [
268 + (json!({}), Default),
269 + (json!({"reasoning_effort": "none"}), Disabled),
270 + (
271 + json!({"reasoning_effort": "high"}),
272 + ReasoningControl::enabled("high"),
273 + ),
274 + (json!({"enable_thinking": true}), Enabled { effort: None }),
275 + (
276 + json!({"enable_thinking": true, "reasoning_effort": "none"}),
277 + Enabled { effort: None },
278 + ),
279 + (
280 + json!({"enable_thinking": true, "reasoning_effort": "low"}),
281 + ReasoningControl::enabled("low"),
282 + ),
283 + (
284 + json!({"enable_thinking": false, "reasoning_effort": "high"}),
285 + Disabled,
286 + ),
287 + (
288 + json!({"thinking": true, "enable_thinking": false}),
289 + Enabled { effort: None },
290 + ),
291 + (
292 + json!({"thinking": false, "enable_thinking": true}),
293 + Disabled,
294 + ),
295 + (
296 + json!({"reasoning_effort": 37}),
297 + ReasoningControl::enabled(EffortValue::Number(37.into())),
298 + ),
299 + (json!({"reasoning_effort": null}), Default),
300 + ] {
301 + let source = kwargs(source);
302 + assert_eq!(
303 + ReasoningControl::from_template_kwargs(&source).unwrap(),
304 + expected,
305 + "{source:?}"
306 + );
307 + }
308 + }
309 +
310 + #[test]
311 + fn request_and_deployment_states_fall_back_before_model_defaults() {
312 + for (request, defaults, expected) in [
313 + (
314 + json!({}),
315 + json!({"enable_thinking": false}),
316 + ReasoningControl::Disabled,
317 + ),
318 + (
319 + json!({"reasoning_effort": "high"}),
320 + json!({"thinking": false}),
321 + ReasoningControl::enabled("high"),
322 + ),
323 + (
324 + json!({"enable_thinking": false}),
325 + json!({"reasoning_effort": "high"}),
326 + ReasoningControl::Disabled,
327 + ),
328 + (
329 + json!({"thinking": true, "reasoning_effort": "none"}),
330 + json!({"reasoning_effort": "low"}),
331 + ReasoningControl::enabled("low"),
332 + ),
333 + (
334 + json!({"thinking": true}),
335 + json!({"thinking": false, "reasoning_effort": "low"}),
336 + ReasoningControl::enabled("max"),
337 + ),
338 + (json!({}), json!({}), ReasoningControl::enabled("max")),
339 + ] {
340 + let request = kwargs(request);
341 + let defaults = kwargs(defaults);
342 + let actual = ReasoningControl::from_template_kwargs(&request)
343 + .unwrap()
344 + .fallback(ReasoningControl::from_template_kwargs(&defaults).unwrap())
345 + .fallback(ReasoningControl::enabled("max"));
346 + assert_eq!(
347 + actual, expected,
348 + "request={request:?}, defaults={defaults:?}"
349 + );
350 + }
351 + }
352 +
353 + #[test]
354 + fn selected_toggle_requires_a_boolean() {
355 + for value in [
356 + json!("true"),
357 + json!(1),
358 + json!(null),
359 + json!({"type": "enabled"}),
360 + ] {
361 + assert!(
362 + ReasoningControl::from_template_kwargs(&kwargs(json!({
363 + "thinking": value, "enable_thinking": true
364 + })))
365 + .unwrap_err()
366 + .is_request_validation_error()
367 + );
368 + }
369 + }
370 +
371 + #[test]
372 + fn invalid_effort_types_are_rejected_only_when_the_source_uses_them() {
373 + for value in [json!(true), json!(false), json!([]), json!({})] {
374 + for toggle in [None, Some(true), Some(false)] {
375 + let mut source = kwargs(json!({"reasoning_effort": value}));
376 + if let Some(toggle) = toggle {
377 + source.insert("thinking".to_owned(), json!(toggle));
378 + }
379 + let control = ReasoningControl::from_template_kwargs(&source);
380 + if toggle == Some(false) {
381 + assert_eq!(control.unwrap(), ReasoningControl::Disabled);
382 + } else {
383 + assert!(control.unwrap_err().is_request_validation_error());
384 + }
385 + }
386 +
387 + // A typed request effort shadows malformed kwargs effort.
388 + let mut request = ChatRequest::for_test();
389 + request.chat_options.reasoning_effort = Some(EffortValue::from("high"));
390 + request.chat_options.template_kwargs = kwargs(json!({"reasoning_effort": value}));
391 + assert_eq!(
392 + ReasoningControl::from_request(&request).unwrap(),
393 + ReasoningControl::enabled("high")
394 + );
395 + }
396 + }
397 +}
rust/src/chat/src/renderer/deepseek.rs
+67 -75
@@ -17,10 +17,10 @@ use serde_json_fmt::JsonFormat;
17 use llm_multimodal::DEEPSEEK_V41_IMAGE_PLACEHOLDER;
18
19 use super::MediaPartSource;
20 +use crate::EffortValue;
21 use crate::error::{Error, Result};
21 -use crate::request::{
22 - ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ReasoningEffort,
23 -};
22 +use crate::reasoning::ReasoningControl;
23 +use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool};
24 use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall};
25
26 const BOS_TOKEN: &str = "<｜begin▁of▁sentence｜>";
@@ -87,21 +87,26 @@ struct RenderedToolSchema<'a> {
87 }
88
89 /// Render one chat request into the final prompt string.
90 -pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Result<String> {
91 - render_request_with_media_order(request, dialect).map(|(prompt, _)| prompt)
90 +pub(super) fn render_request(
91 + request: &ChatRequest,
92 + dialect: DsDialect,
93 + reasoning: &ReasoningControl,
94 +) -> Result<String> {
95 + render_request_with_media_order(request, dialect, reasoning).map(|(prompt, _)| prompt)
96 }
97
98 /// Render one chat request and record media in placeholder order.
99 pub(super) fn render_request_with_media_order(
100 request: &ChatRequest,
101 dialect: DsDialect,
102 + reasoning: &ReasoningControl,
103 ) -> Result<(String, Vec<MediaPartSource>)> {
99 - let (thinking_mode, reasoning_effort_prompt) = match dialect {
100 - DsDialect::V4 => {
101 - resolve_thinking_options(request).map(|(mode, prompt)| (mode, Cow::Borrowed(prompt)))?
102 - }
103 - DsDialect::V41 => resolve_v41_thinking_options(request)?,
104 + let thinking_mode = if reasoning.is_enabled() {
105 + ThinkingMode::Thinking
106 + } else {
107 + ThinkingMode::Chat
108 };
109 + let reasoning_effort_prompt = reasoning_prompt(reasoning, dialect);
110 let request_tools = request_tools(request);
111 let synthetic_tool_system = needs_synthetic_tool_system(request, request_tools);
112 let drop_thinking = request.parse_template_bool("drop_thinking")?.unwrap_or(true)
@@ -254,76 +259,63 @@ pub(super) fn render_request_with_media_order(
259 Ok((out, media_order))
260 }
261
257 -/// Resolve DeepSeek V4's thinking controls. Unlike the Python tokenizer
258 -/// wrapper, the Rust renderer only consumes the typed top-level
259 -/// `reasoning_effort`; the generic template-kwargs map is left for HF
260 -/// templates.
261 -fn resolve_thinking_options(request: &ChatRequest) -> Result<(ThinkingMode, &'static str)> {
262 - let mut thinking_mode = match request.enable_thinking()?.unwrap_or(true) {
263 - true => ThinkingMode::Thinking,
264 - false => ThinkingMode::Chat,
265 - };
266 - let mut reasoning_effort_prompt = REASONING_EFFORT_HIGH;
267 -
268 - match request.chat_options.reasoning_effort {
269 - Some(ReasoningEffort::None) => thinking_mode = ThinkingMode::Chat,
270 - Some(ReasoningEffort::Max) => {
271 - reasoning_effort_prompt = REASONING_EFFORT_MAX;
272 - }
273 - Some(ReasoningEffort::XHigh | ReasoningEffort::High) => {
274 - reasoning_effort_prompt = REASONING_EFFORT_HIGH;
262 +/// Map the shared reasoning decision to the selected DeepSeek effort dialect.
263 +pub(super) fn resolve_reasoning(
264 + control: ReasoningControl,
265 + dialect: DsDialect,
266 +) -> Result<ReasoningControl> {
267 + let control = control.fallback(ReasoningControl::enabled("high"));
268 + if !control.is_enabled() {
269 + return Ok(control);
270 + }
271 + let effort = control.effort().expect("enabled DeepSeek reasoning has a default effort");
272 + let effort = match dialect {
273 + DsDialect::V4 => {
274 + let effort = match effort.as_str() {
275 + Some("minimal" | "medium" | "low") => "low",
276 + Some("high" | "xhigh") => "high",
277 + Some("max") => "max",
278 + _ => {
279 + return Err(Error::InvalidReasoningEffort(format!(
280 + "DeepSeek V4 reasoning_effort must be minimal, low, medium, high, xhigh, or max, got {effort}"
281 + )));
282 + }
283 + };
284 + EffortValue::from(effort)
285 }
276 - Some(ReasoningEffort::Minimal | ReasoningEffort::Medium | ReasoningEffort::Low) => {
277 - reasoning_effort_prompt = "";
286 + DsDialect::V41 => {
287 + let budget = match effort {
288 + EffortValue::String(effort) => match effort.as_str() {
289 + "low" => 25,
290 + "high" => 50,
291 + "xhigh" => 75,
292 + "max" => 100,
293 + _ => return Err(invalid_v41_reasoning_effort()),
294 + },
295 + EffortValue::Number(budget) => budget
296 + .as_u64()
297 + .filter(|budget| (1..=100).contains(budget))
298 + .ok_or_else(invalid_v41_reasoning_effort)?,
299 + };
300 + EffortValue::Number(budget.into())
301 }
279 - None => {}
280 - }
281 -
282 - Ok((thinking_mode, reasoning_effort_prompt))
302 + };
303 + Ok(ReasoningControl::enabled(effort))
304 }
305
285 -/// Resolve V4.1's numeric reasoning effort using the top-level value before template kwargs.
286 -fn resolve_v41_thinking_options(
287 - request: &ChatRequest,
288 -) -> Result<(ThinkingMode, Cow<'static, str>)> {
289 - let mut thinking = request.enable_thinking()?.unwrap_or(true);
290 - let budget = match request.chat_options.reasoning_effort {
291 - Some(ReasoningEffort::None) => {
292 - thinking = false;
293 - 50
294 - }
295 - Some(ReasoningEffort::Low) => 25,
296 - Some(ReasoningEffort::High) => 50,
297 - Some(ReasoningEffort::XHigh) => 75,
298 - Some(ReasoningEffort::Max) => 100,
299 - Some(ReasoningEffort::Minimal | ReasoningEffort::Medium) => {
300 - return Err(invalid_v41_reasoning_effort());
301 - }
302 - None => match request.chat_options.template_kwargs.get("reasoning_effort") {
303 - None | Some(Value::Null) => 50,
304 - Some(Value::String(effort)) => match effort.as_str() {
305 - "low" => 25,
306 - "high" => 50,
307 - "xhigh" => 75,
308 - "max" => 100,
309 - _ => return Err(invalid_v41_reasoning_effort()),
310 - },
311 - Some(Value::Number(budget)) => budget
312 - .as_u64()
313 - .filter(|budget| (1..=100).contains(budget))
314 - .ok_or_else(invalid_v41_reasoning_effort)?,
315 - Some(_) => return Err(invalid_v41_reasoning_effort()),
316 - },
306 +fn reasoning_prompt(control: &ReasoningControl, dialect: DsDialect) -> Cow<'static, str> {
307 + let Some(effort) = control.effort() else {
308 + return Cow::Borrowed("");
309 };
318 - if thinking {
319 - Ok((
320 - ThinkingMode::Thinking,
321 - Cow::Owned(format!(
322 - "Reasoning Effort: {budget} (range 1-100, the higher the value, the more thorough the reasoning)\n\n"
323 - )),
324 - ))
325 - } else {
326 - Ok((ThinkingMode::Chat, Cow::Borrowed("")))
310 + match dialect {
311 + DsDialect::V4 => Cow::Borrowed(match effort.as_str() {
312 + Some("high") => REASONING_EFFORT_HIGH,
313 + Some("max") => REASONING_EFFORT_MAX,
314 + _ => "",
315 + }),
316 + DsDialect::V41 => Cow::Owned(format!(
317 + "Reasoning Effort: {effort} (range 1-100, the higher the value, the more thorough the reasoning)\n\n"
318 + )),
319 }
320 }
321
rust/src/chat/src/renderer/deepseek_v32/encoding.rs
+6 -2
@@ -14,6 +14,7 @@ use serde_json::Value;
14 use serde_json_fmt::JsonFormat;
15
16 use crate::error::{Error, Result};
17 +use crate::reasoning::ReasoningControl;
18 use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatRole, ChatTool};
19 use crate::{AssistantContentBlock, AssistantMessageExt, AssistantToolCall};
20
@@ -42,8 +43,11 @@ struct RenderedToolSchema<'a> {
43 }
44
45 /// Render one chat request into the final prompt string.
45 -pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
46 - let thinking_mode = match request.enable_thinking()?.unwrap_or(false) {
46 +pub(super) fn render_request(
47 + request: &ChatRequest,
48 + reasoning: &ReasoningControl,
49 +) -> Result<String> {
50 + let thinking_mode = match reasoning.is_enabled() {
51 true => ThinkingMode::Thinking,
52 false => ThinkingMode::Chat,
53 };
rust/src/chat/src/renderer/deepseek_v32/mod.rs
+17 -7
@@ -3,31 +3,41 @@
3
4 mod encoding;
5
6 +use std::collections::HashMap;
7 +
8 +use serde_json::Value;
9 use vllm_text::Prompt;
10
8 -use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
11 +use super::{ChatRenderer, RenderedPrompt};
12 use crate::Result;
13 +use crate::reasoning::ReasoningControl;
14 use crate::request::ChatRequest;
15
16 /// Dedicated DeepSeek V3.2 renderer.
13 -#[derive(Debug, Clone, Copy, Default)]
14 -pub struct DeepSeekV32ChatRenderer;
17 +#[derive(Debug, Clone, Default)]
18 +pub struct DeepSeekV32ChatRenderer {
19 + default_template_kwargs: HashMap<String, Value>,
20 +}
21
22 impl DeepSeekV32ChatRenderer {
23 /// Create the dedicated DeepSeek V3.2 renderer.
18 - pub fn new() -> Self {
19 - Self
24 + pub fn new(default_template_kwargs: HashMap<String, Value>) -> Self {
25 + Self {
26 + default_template_kwargs,
27 + }
28 }
29 }
30
31 impl ChatRenderer for DeepSeekV32ChatRenderer {
32 fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
33 request.validate()?;
34 + let reasoning = ReasoningControl::resolve(request, &self.default_template_kwargs)?
35 + .fallback(ReasoningControl::Disabled);
36
37 Ok(RenderedPrompt {
28 - prompt: Prompt::Text(encoding::render_request(request)?),
38 + prompt: Prompt::Text(encoding::render_request(request, &reasoning)?),
39 media_order: None,
30 - effective_template_kwargs: request_template_kwargs(request),
40 + effective_template_kwargs: reasoning.template_kwargs(request),
41 })
42 }
43 }
rust/src/chat/src/renderer/deepseek_v32/tests.rs
+26 -3
@@ -8,6 +8,7 @@ use serde_json::{Value, json};
8 use thiserror_ext::AsReport;
9
10 use super::DeepSeekV32ChatRenderer;
11 +use crate::EffortValue;
12 use crate::error::Error;
13 use crate::event::{AssistantContentBlock, AssistantToolCall};
14 use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
@@ -18,7 +19,7 @@ use crate::request::{
19 use crate::{ChatRenderer, ChatRole};
20
21 fn render_request(request: &ChatRequest) -> String {
21 - DeepSeekV32ChatRenderer::new()
22 + DeepSeekV32ChatRenderer::default()
23 .render(request)
24 .unwrap()
25 .prompt
@@ -26,8 +27,30 @@ fn render_request(request: &ChatRequest) -> String {
27 .expect("deepseek renderer should return text prompt")
28 }
29
30 +#[test]
31 +fn effort_enables_binary_thinking_and_none_disables_it() {
32 + let mut request = ChatRequest::for_test();
33 + for (effort, enabled) in [
34 + (EffortValue::from("high"), true),
35 + (EffortValue::from("custom"), true),
36 + (EffortValue::Number(42.into()), true),
37 + (EffortValue::from("none"), false),
38 + ] {
39 + request.chat_options.reasoning_effort = Some(effort);
40 + let rendered = DeepSeekV32ChatRenderer::default().render(&request).unwrap();
41 + assert_eq!(
42 + rendered.prompt.into_text().unwrap().ends_with("<think>"),
43 + enabled
44 + );
45 + assert_eq!(
46 + rendered.effective_template_kwargs["enable_thinking"],
47 + enabled
48 + );
49 + }
50 +}
51 +
52 fn render_result(request: &ChatRequest) -> Result<String, Error> {
30 - DeepSeekV32ChatRenderer::new().render(request).map(|rendered| {
53 + DeepSeekV32ChatRenderer::default().render(request).map(|rendered| {
54 rendered
55 .prompt
56 .into_text()
@@ -345,7 +368,7 @@ fn render_rejects_multimodal_input() {
368 ..ChatRequest::for_test()
369 };
370
348 - let error = DeepSeekV32ChatRenderer::new().render(&request).unwrap_err();
371 + let error = DeepSeekV32ChatRenderer::default().render(&request).unwrap_err();
372
373 assert!(matches!(
374 error,
rust/src/chat/src/renderer/deepseek_v4/mod.rs
+23 -7
@@ -1,31 +1,47 @@
1 // SPDX-License-Identifier: Apache-2.0
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +use std::collections::HashMap;
5 +
6 +use serde_json::Value;
7 use vllm_text::Prompt;
8
9 use super::deepseek::{self, DsDialect};
7 -use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
10 +use super::{ChatRenderer, RenderedPrompt};
11 use crate::Result;
12 +use crate::reasoning::ReasoningControl;
13 use crate::request::ChatRequest;
14
15 /// Dedicated DeepSeek V4 renderer.
12 -#[derive(Debug, Clone, Copy, Default)]
13 -pub struct DeepSeekV4ChatRenderer;
16 +#[derive(Debug, Clone, Default)]
17 +pub struct DeepSeekV4ChatRenderer {
18 + default_template_kwargs: HashMap<String, Value>,
19 +}
20
21 impl DeepSeekV4ChatRenderer {
16 - pub fn new() -> Self {
17 - Self
22 + pub fn new(default_template_kwargs: HashMap<String, Value>) -> Self {
23 + Self {
24 + default_template_kwargs,
25 + }
26 }
27 }
28
29 impl ChatRenderer for DeepSeekV4ChatRenderer {
30 fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
31 request.validate()?;
32 + let reasoning = deepseek::resolve_reasoning(
33 + ReasoningControl::resolve(request, &self.default_template_kwargs)?,
34 + DsDialect::V4,
35 + )?;
36
37 Ok(RenderedPrompt {
26 - prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V4)?),
38 + prompt: Prompt::Text(deepseek::render_request(
39 + request,
40 + DsDialect::V4,
41 + &reasoning,
42 + )?),
43 media_order: None,
28 - effective_template_kwargs: request_template_kwargs(request),
44 + effective_template_kwargs: reasoning.template_kwargs(request),
45 })
46 }
47 }
rust/src/chat/src/renderer/deepseek_v4/tests.rs
+62 -22
@@ -8,13 +8,14 @@ use serde_json::Value;
8
9 use super::DeepSeekV4ChatRenderer;
10 use crate::ChatRenderer;
11 +use crate::EffortValue;
12 use crate::event::{AssistantContentBlock, AssistantToolCall};
13 use crate::renderer::deepseek_v41::DeepSeekV41ChatRenderer;
14 use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
14 -use crate::request::{ChatMessage, ChatRequest, ChatTool, GenerationPromptMode, ReasoningEffort};
15 +use crate::request::{ChatMessage, ChatRequest, ChatTool, GenerationPromptMode};
16
17 fn render_request(request: &ChatRequest) -> String {
17 - DeepSeekV4ChatRenderer::new()
18 + DeepSeekV4ChatRenderer::default()
19 .render(request)
20 .unwrap()
21 .prompt
@@ -31,13 +32,13 @@ fn thinking_request(messages: Vec<ChatMessage>) -> ChatRequest {
32 .chat_options
33 .template_kwargs
34 .insert("thinking".to_string(), Value::Bool(true));
34 - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
35 + request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
36 request
37 }
38
39 fn fixture_request(input_name: &str) -> ChatRequest {
40 let mut request = fixture_chat_request(&fixture_path(input_name), deepseek_fixture_options());
40 - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
41 + request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
42 request
43 }
44
@@ -203,7 +204,7 @@ fn reasoning_effort_max_adds_prefix_when_thinking_is_enabled() {
204 .chat_options
205 .template_kwargs
206 .insert("thinking".to_string(), Value::Bool(true));
206 - request.chat_options.reasoning_effort = Some(ReasoningEffort::Max);
207 + request.chat_options.reasoning_effort = Some(EffortValue::from("max"));
208
209 let rendered = render_request(&request);
210
@@ -226,7 +227,7 @@ fn reasoning_effort_high_adds_0731_high_prefix() {
227 .chat_options
228 .template_kwargs
229 .insert("thinking".to_string(), Value::Bool(true));
229 - request.chat_options.reasoning_effort = Some(ReasoningEffort::High);
230 + request.chat_options.reasoning_effort = Some(EffortValue::from("high"));
231
232 let rendered = render_request(&request);
233
@@ -258,7 +259,7 @@ fn reasoning_effort_xhigh_maps_to_high() {
259 messages: vec![ChatMessage::user("solve it")],
260 ..ChatRequest::for_test()
261 };
261 - request.chat_options.reasoning_effort = Some(ReasoningEffort::XHigh);
262 + request.chat_options.reasoning_effort = Some(EffortValue::from("xhigh"));
263
264 let rendered = render_request(&request);
265
@@ -268,7 +269,7 @@ fn reasoning_effort_xhigh_maps_to_high() {
269
270 #[test]
271 fn minimal_and_medium_reasoning_effort_map_to_low() {
271 - for effort in [ReasoningEffort::Minimal, ReasoningEffort::Medium] {
272 + for effort in [EffortValue::from("minimal"), EffortValue::from("medium")] {
273 let mut request = ChatRequest {
274 messages: vec![ChatMessage::user("solve it")],
275 ..ChatRequest::for_test()
@@ -292,7 +293,7 @@ fn reasoning_effort_low_keeps_the_default_prompt() {
293 .chat_options
294 .template_kwargs
295 .insert("thinking".to_string(), Value::Bool(true));
295 - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
296 + request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
297
298 let rendered = render_request(&request);
299
@@ -305,11 +306,7 @@ fn reasoning_effort_none_disables_thinking() {
306 messages: vec![ChatMessage::user("answer directly")],
307 ..ChatRequest::for_test()
308 };
308 - request
309 - .chat_options
310 - .template_kwargs
311 - .insert("thinking".to_string(), Value::Bool(true));
312 - request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
309 + request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
310
311 let rendered = render_request(&request);
312
@@ -317,6 +314,49 @@ fn reasoning_effort_none_disables_thinking() {
314 .assert_eq(&rendered);
315 }
316
317 +#[test]
318 +fn normalized_reasoning_drives_both_prompt_and_compatibility_kwargs() {
319 + for (kwargs, enabled, effort, prefix) in [
320 + (
321 + serde_json::json!({"enable_thinking": true, "reasoning_effort": "none"}),
322 + true,
323 + "high",
324 + "Absolute maximum",
325 + ),
326 + (
327 + serde_json::json!({"thinking": false, "enable_thinking": true, "reasoning_effort": "max"}),
328 + false,
329 + "none",
330 + "",
331 + ),
332 + (
333 + serde_json::json!({"reasoning_effort": "xhigh"}),
334 + true,
335 + "high",
336 + "Absolute maximum",
337 + ),
338 + ] {
339 + let mut request = ChatRequest::for_test();
340 + request.chat_options.template_kwargs = serde_json::from_value(kwargs).unwrap();
341 + let rendered = DeepSeekV4ChatRenderer::default().render(&request).unwrap();
342 + let prompt = rendered.prompt.into_text().unwrap();
343 + assert_eq!(prompt.ends_with("<think>"), enabled);
344 + assert_eq!(prompt.contains("Reasoning Effort:"), enabled);
345 + if enabled {
346 + assert!(prompt.contains(prefix));
347 + }
348 + assert_eq!(rendered.effective_template_kwargs["thinking"], enabled);
349 + assert_eq!(
350 + rendered.effective_template_kwargs["enable_thinking"],
351 + enabled
352 + );
353 + assert_eq!(
354 + rendered.effective_template_kwargs["reasoning_effort"],
355 + effort
356 + );
357 + }
358 +}
359 +
360 #[test]
361 fn system_to_assistant_transitions_in_chat_and_thinking_modes() {
362 let rendered = [false, true].map(|enable_thinking| {
@@ -338,7 +378,7 @@ fn system_to_assistant_transitions_in_chat_and_thinking_modes() {
378 .chat_options
379 .template_kwargs
380 .insert("enable_thinking".to_string(), Value::Bool(enable_thinking));
341 - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
381 + request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
382 render_request(&request)
383 });
384
@@ -366,7 +406,7 @@ fn trailing_system_transitions_in_chat_and_thinking_modes() {
406 .chat_options
407 .template_kwargs
408 .insert("enable_thinking".to_string(), Value::Bool(enable_thinking));
369 - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
409 + request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
410 render_request(&request)
411 });
412
@@ -380,7 +420,7 @@ fn trailing_system_transitions_in_chat_and_thinking_modes() {
420 }
421
422 #[test]
383 -fn reasoning_effort_template_kwarg_is_ignored() {
423 +fn reasoning_effort_template_kwarg_selects_effort() {
424 let mut request = ChatRequest {
425 messages: vec![ChatMessage::user("solve it")],
426 ..ChatRequest::for_test()
@@ -396,15 +436,15 @@ fn reasoning_effort_template_kwarg_is_ignored() {
436
437 let rendered = render_request(&request);
438
399 - assert!(rendered.starts_with("<｜begin▁of▁sentence｜>Reasoning Effort: Absolute maximum"));
439 + assert!(rendered.starts_with("<｜begin▁of▁sentence｜>Reasoning Effort: Beyond maximum"));
440 assert!(rendered.ends_with("<｜Assistant｜><think>"));
441 }
442
443 #[test]
444 fn tool_call_arguments_preserve_non_objects_like_recipe() {
445 let renderers: [(&dyn ChatRenderer, &str); 2] = [
406 - (&DeepSeekV4ChatRenderer::new(), "parameter"),
407 - (&DeepSeekV41ChatRenderer::new(), " parameter"),
446 + (&DeepSeekV4ChatRenderer::default(), "parameter"),
447 + (&DeepSeekV41ChatRenderer::default(), " parameter"),
448 ];
449 for arguments in [
450 "not json",
@@ -600,7 +640,7 @@ fn drop_thinking_false_keeps_prior_assistant_reasoning() {
640 .chat_options
641 .template_kwargs
642 .insert("drop_thinking".to_string(), Value::Bool(false));
603 - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
643 + request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
644
645 let rendered = render_request(&request);
646
@@ -693,7 +733,7 @@ fn trailing_system_omits_generation_prompt_when_disabled() {
733 .chat_options
734 .template_kwargs
735 .insert("enable_thinking".to_string(), Value::Bool(false));
696 - request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
736 + request.chat_options.reasoning_effort = Some(EffortValue::from("low"));
737
738 let rendered = render_request(&request);
739
rust/src/chat/src/renderer/deepseek_v41/mod.rs
+19 -7
@@ -1,33 +1,45 @@
1 // SPDX-License-Identifier: Apache-2.0
2 // SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 +use std::collections::HashMap;
5 +
6 +use serde_json::Value;
7 use vllm_text::Prompt;
8
9 use super::deepseek::{self, DsDialect};
7 -use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
10 +use super::{ChatRenderer, RenderedPrompt};
11 use crate::Result;
12 +use crate::reasoning::ReasoningControl;
13 use crate::request::ChatRequest;
14
15 /// Dedicated DeepSeek V4.1 renderer.
12 -#[derive(Debug, Clone, Copy, Default)]
13 -pub struct DeepSeekV41ChatRenderer;
16 +#[derive(Debug, Clone, Default)]
17 +pub struct DeepSeekV41ChatRenderer {
18 + default_template_kwargs: HashMap<String, Value>,
19 +}
20
21 impl DeepSeekV41ChatRenderer {
16 - pub fn new() -> Self {
17 - Self
22 + pub fn new(default_template_kwargs: HashMap<String, Value>) -> Self {
23 + Self {
24 + default_template_kwargs,
25 + }
26 }
27 }
28
29 impl ChatRenderer for DeepSeekV41ChatRenderer {
30 fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
31 request.validate()?;
32 + let reasoning = deepseek::resolve_reasoning(
33 + ReasoningControl::resolve(request, &self.default_template_kwargs)?,
34 + DsDialect::V41,
35 + )?;
36 let (prompt, media_order) =
25 - deepseek::render_request_with_media_order(request, DsDialect::V41)?;
37 + deepseek::render_request_with_media_order(request, DsDialect::V41, &reasoning)?;
38
39 Ok(RenderedPrompt {
40 prompt: Prompt::Text(prompt),
41 media_order: Some(media_order),
30 - effective_template_kwargs: request_template_kwargs(request),
42 + effective_template_kwargs: reasoning.template_kwargs(request),
43 })
44 }
45 }
rust/src/chat/src/renderer/deepseek_v41/tests.rs
+49 -15
@@ -8,14 +8,15 @@ use serde_json::{Value, json};
8
9 use super::DeepSeekV41ChatRenderer;
10 use crate::ChatRenderer;
11 +use crate::EffortValue;
12 use crate::event::{AssistantContentBlock, AssistantToolCall};
13 use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
14 use crate::request::{
14 - ChatContent, ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort,
15 + ChatContent, ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode,
16 };
17
18 fn render(request: &ChatRequest) -> String {
18 - DeepSeekV41ChatRenderer::new()
19 + DeepSeekV41ChatRenderer::default()
20 .render(request)
21 .unwrap()
22 .prompt
@@ -64,7 +65,7 @@ fn inlines_image_placeholder_at_image_part_positions() {
65 // The placeholder is unconditional (matching the Python encoding's
66 // `IMAGE_PLACEHOLDER`); requests to a multimodal-less backend fail later,
67 // at media preparation.
67 - let renderer = DeepSeekV41ChatRenderer::new();
68 + let renderer = DeepSeekV41ChatRenderer::default();
69 let request = ChatRequest {
70 messages: vec![ChatMessage::User {
71 content: ChatContent::Parts(vec![
@@ -87,7 +88,7 @@ fn inlines_image_placeholder_at_image_part_positions() {
88
89 #[test]
90 fn media_order_follows_reordered_tool_results() {
90 - let renderer = DeepSeekV41ChatRenderer::new();
91 + let renderer = DeepSeekV41ChatRenderer::default();
92 let request = ChatRequest {
93 messages: vec![
94 ChatMessage::assistant_blocks(vec![
@@ -136,7 +137,7 @@ fn media_order_follows_reordered_tool_results() {
137
138 #[test]
139 fn media_order_excludes_dropped_historical_developer_content() {
139 - let renderer = DeepSeekV41ChatRenderer::new();
140 + let renderer = DeepSeekV41ChatRenderer::default();
141 let request = ChatRequest {
142 messages: vec![
143 ChatMessage::developer(vec![image_part("historical-image")], None),
@@ -165,10 +166,11 @@ fn image_part(uuid: &str) -> ChatContentPart {
166 fn maps_reasoning_effort_to_reference_numeric_budget() {
167 for (effort, budget) in [
168 (None, 50),
168 - (Some(ReasoningEffort::Low), 25),
169 - (Some(ReasoningEffort::High), 50),
170 - (Some(ReasoningEffort::XHigh), 75),
171 - (Some(ReasoningEffort::Max), 100),
169 + (Some(EffortValue::from("low")), 25),
170 + (Some(EffortValue::from("high")), 50),
171 + (Some(EffortValue::from("xhigh")), 75),
172 + (Some(EffortValue::from("max")), 100),
173 + (Some(EffortValue::Number(37.into())), 37),
174 ] {
175 let mut request = request();
176 request.chat_options.reasoning_effort = effort;
@@ -193,11 +195,43 @@ fn accepts_numeric_template_effort_with_top_level_precedence() {
195 request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort);
196 assert!(render(&request).contains(&format!("Reasoning Effort: {budget} (range 1-100")));
197
196 - request.chat_options.reasoning_effort = Some(ReasoningEffort::XHigh);
198 + request.chat_options.reasoning_effort = Some(EffortValue::from("xhigh"));
199 assert!(render(&request).contains("Reasoning Effort: 75 (range 1-100"));
200 }
201 }
202
203 +#[test]
204 +fn enabling_with_none_inherits_deployment_effort_and_reports_numeric_control() {
205 + let renderer =
206 + DeepSeekV41ChatRenderer::new([("reasoning_effort".to_string(), json!(37))].into());
207 + let mut request = request();
208 + request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
209 + request
210 + .chat_options
211 + .template_kwargs
212 + .insert("enable_thinking".into(), json!(true));
213 + let rendered = renderer.render(&request).unwrap();
214 + let prompt = rendered.prompt.into_text().unwrap();
215 + assert!(prompt.contains("Reasoning Effort: 37 (range 1-100"));
216 + assert!(prompt.ends_with("<think>"));
217 + assert_eq!(rendered.effective_template_kwargs["reasoning_effort"], 37);
218 + assert_eq!(rendered.effective_template_kwargs["enable_thinking"], true);
219 +
220 + request.chat_options.template_kwargs.insert("thinking".into(), json!(false));
221 + request
222 + .chat_options
223 + .template_kwargs
224 + .insert("reasoning_effort".into(), json!(101));
225 + request.chat_options.reasoning_effort = None;
226 + let rendered = renderer.render(&request).unwrap();
227 + assert!(rendered.prompt.into_text().unwrap().ends_with("</think>"));
228 + assert_eq!(
229 + rendered.effective_template_kwargs["reasoning_effort"],
230 + "none"
231 + );
232 + assert_eq!(rendered.effective_template_kwargs["enable_thinking"], false);
233 +}
234 +
235 #[test]
236 fn rejects_undefined_effort_names_and_invalid_numeric_budgets() {
237 for effort in [
@@ -211,13 +245,13 @@ fn rejects_undefined_effort_names_and_invalid_numeric_budgets() {
245 ] {
246 let mut request = request();
247 request.chat_options.template_kwargs.insert("reasoning_effort".into(), effort);
214 - let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err();
248 + let error = DeepSeekV41ChatRenderer::default().render(&request).unwrap_err();
249 assert!(error.is_request_validation_error());
250 }
217 - for effort in [ReasoningEffort::Minimal, ReasoningEffort::Medium] {
251 + for effort in [EffortValue::from("minimal"), EffortValue::from("medium")] {
252 let mut request = request();
253 request.chat_options.reasoning_effort = Some(effort);
220 - let error = DeepSeekV41ChatRenderer::new().render(&request).unwrap_err();
254 + let error = DeepSeekV41ChatRenderer::default().render(&request).unwrap_err();
255 assert!(error.is_request_validation_error());
256 }
257 }
@@ -229,8 +263,8 @@ fn chat_mode_and_none_effort_use_closed_thinking_prefix() {
263 let expected = expect!["<｜begin▁of▁sentence｜><｜User｜>question<｜Assistant｜></think>"];
264 expected.assert_eq(&render(&request));
265
232 - request.chat_options.template_kwargs.insert("thinking".into(), json!(true));
233 - request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
266 + request.chat_options.template_kwargs.remove("thinking");
267 + request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
268 expected.assert_eq(&render(&request));
269 }
270
rust/src/chat/src/renderer/harmony/mod.rs
+51 -27
@@ -5,21 +5,26 @@
5
6 pub(crate) mod encoding;
7
8 +use std::collections::HashMap;
9 +
10 use openai_harmony::HarmonyEncoding;
11 use openai_harmony::chat::{
12 Author, Conversation, DeveloperContent, Message, ReasoningEffort as HarmonyReasoningEffort,
13 Role, SystemContent, ToolDescription,
14 };
15 +use serde_json::Value;
16 use thiserror_ext::AsReport as _;
17 use time::macros::format_description;
18 use vllm_text::Prompt;
19
20 use self::encoding::harmony_encoding;
18 -use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
21 +use super::{ChatRenderer, RenderedPrompt};
22 +use crate::AssistantMessageExt as _;
23 +use crate::EffortValue;
24 use crate::error::{Error, Result};
25 use crate::event::AssistantContentBlock;
26 +use crate::reasoning::ReasoningControl;
27 use crate::request::{ChatContent, ChatMessage, ChatRequest, ChatTool, GenerationPromptMode};
22 -use crate::{AssistantMessageExt as _, ReasoningEffort};
28
29 const SYSTEM_START_DATE_ENV: &str = "VLLM_SYSTEM_START_DATE";
30 const HARMONY_SYSTEM_INSTRUCTIONS_ENV: &str = "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS";
@@ -28,6 +33,7 @@ const HARMONY_SYSTEM_INSTRUCTIONS_ENV: &str = "VLLM_GPT_OSS_HARMONY_SYSTEM_INSTR
33 pub struct HarmonyChatRenderer {
34 encoding: &'static HarmonyEncoding,
35 options: Options,
36 + default_template_kwargs: HashMap<String, Value>,
37 }
38
39 struct Options {
@@ -44,8 +50,9 @@ impl HarmonyChatRenderer {
50 /// unset, the renderer uses the current local date with a UTC fallback.
51 /// - `VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS` moves leading instructions
52 /// into the system model identity when set to a non-zero integer.
47 - pub fn new() -> Result<Self> {
53 + pub fn new(default_template_kwargs: HashMap<String, Value>) -> Result<Self> {
54 Self::with_options(
55 + default_template_kwargs,
56 env_system_start_date(),
57 env_use_harmony_system_instructions(),
58 )
@@ -57,11 +64,13 @@ impl HarmonyChatRenderer {
64 /// Production code should call [`Self::new`] so the renderer observes the
65 /// same environment contract as the Python Harmony path.
66 pub fn with_options(
67 + default_template_kwargs: HashMap<String, Value>,
68 system_start_date: impl Into<String>,
69 use_system_instructions: bool,
70 ) -> Result<Self> {
71 Ok(Self {
72 encoding: harmony_encoding()?,
73 + default_template_kwargs,
74 options: Options {
75 system_start_date: system_start_date.into(),
76 use_system_instructions,
@@ -73,7 +82,11 @@ impl HarmonyChatRenderer {
82 ///
83 /// Harmony owns both prompt formatting and tokenization, so the Rust
84 /// frontend bypasses the generic HF tokenizer path for GPT-OSS input.
76 - fn render_token_ids(&self, request: &ChatRequest) -> Result<Vec<u32>> {
85 + fn render_token_ids(
86 + &self,
87 + request: &ChatRequest,
88 + effort: HarmonyReasoningEffort,
89 + ) -> Result<Vec<u32>> {
90 if request.has_multimodal() {
91 return Err(Error::UnsupportedMultimodalContent("image_url"));
92 }
@@ -86,7 +99,8 @@ impl HarmonyChatRenderer {
99 ));
100 }
101
89 - let messages = auto_drop_analysis_messages(to_harmony_messages(request, &self.options)?);
102 + let messages =
103 + auto_drop_analysis_messages(to_harmony_messages(request, &self.options, effort)?);
104 let conversation = Conversation::from_messages(messages);
105 // Pass `None` so oss-harmony does not apply its narrower built-in
106 // analysis-drop policy after the Rust-side Python-parity cleanup above.
@@ -114,10 +128,13 @@ impl ChatRenderer for HarmonyChatRenderer {
128 /// Render a chat request as [`Prompt::TokenIds`] with template kwargs echoed
129 /// for downstream accounting/debugging.
130 fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
131 + let reasoning = ReasoningControl::resolve(request, &self.default_template_kwargs)?
132 + .fallback(ReasoningControl::enabled("medium"));
133 + let effort = to_harmony_reasoning_effort(reasoning.effort())?;
134 Ok(RenderedPrompt {
118 - prompt: Prompt::TokenIds(self.render_token_ids(request)?),
135 + prompt: Prompt::TokenIds(self.render_token_ids(request, effort)?),
136 media_order: None,
120 - effective_template_kwargs: request_template_kwargs(request),
137 + effective_template_kwargs: reasoning.template_kwargs(request),
138 })
139 }
140 }
@@ -127,12 +144,21 @@ impl ChatRenderer for HarmonyChatRenderer {
144 /// This adds the Harmony system/developer preamble, peels at most one leading
145 /// system/developer instruction message, and then lowers the remaining chat
146 /// history message-by-message.
130 -fn to_harmony_messages(request: &ChatRequest, options: &Options) -> Result<Vec<Message>> {
147 +fn to_harmony_messages(
148 + request: &ChatRequest,
149 + options: &Options,
150 + effort: HarmonyReasoningEffort,
151 +) -> Result<Vec<Message>> {
152 let (instructions, leading_developer_tools, remaining_messages) =
153 peel_leading_instructions(&request.messages)?;
154 let tool_call_names = tool_call_names(&request.messages);
134 - let mut messages =
135 - build_harmony_preamble(request, instructions, leading_developer_tools, options)?;
155 + let mut messages = build_harmony_preamble(
156 + request,
157 + instructions,
158 + leading_developer_tools,
159 + options,
160 + effort,
161 + )?;
162
163 for message in remaining_messages {
164 messages.extend(to_harmony_message(message, &tool_call_names, options)?);
@@ -177,12 +203,13 @@ fn build_harmony_preamble(
203 instructions: Option<String>,
204 leading_developer_tools: Option<&[ChatTool]>,
205 options: &Options,
206 + effort: HarmonyReasoningEffort,
207 ) -> Result<Vec<Message>> {
208 let mut messages = vec![Message::from_role_and_content(
209 Role::System,
210 system_content(
211 instructions.as_deref().filter(|_| options.use_system_instructions),
185 - request.chat_options.reasoning_effort,
212 + Some(effort),
213 &options.system_start_date,
214 )?,
215 )];
@@ -227,18 +254,18 @@ fn preamble_tool_descriptions(
254
255 /// Construct the Harmony system content for the request preamble.
256 ///
230 -/// Harmony defaults the reasoning effort to `medium` when none is provided, so
231 -/// this only sets an explicit effort after validating vLLM's request value.
257 +/// The preamble receives normalized effort; later system messages use Harmony's
258 +/// own default when their effort is omitted.
259 fn system_content(
260 instructions: Option<&str>,
234 - reasoning_effort: Option<ReasoningEffort>,
261 + reasoning_effort: Option<HarmonyReasoningEffort>,
262 system_start_date: &str,
263 ) -> Result<SystemContent> {
264 let mut content =
265 SystemContent::new().with_conversation_start_date(system_start_date.to_string());
266
267 if let Some(reasoning_effort) = reasoning_effort {
241 - content = content.with_reasoning_effort(to_harmony_reasoning_effort(reasoning_effort)?);
268 + content = content.with_reasoning_effort(reasoning_effort);
269 }
270
271 if let Some(instructions) = instructions.filter(|text| !text.is_empty()) {
@@ -445,20 +472,17 @@ fn to_tool_descriptions(tools: &[ChatTool]) -> Vec<ToolDescription> {
472 .collect()
473 }
474
448 -/// Map supported OpenAI reasoning-effort values onto Harmony's enum.
475 +/// Map supported reasoning-effort values onto Harmony's enum.
476 fn to_harmony_reasoning_effort(
450 - reasoning_effort: ReasoningEffort,
477 + reasoning_effort: Option<&EffortValue>,
478 ) -> Result<HarmonyReasoningEffort> {
452 - match reasoning_effort {
453 - ReasoningEffort::Low => Ok(HarmonyReasoningEffort::Low),
454 - ReasoningEffort::Medium => Ok(HarmonyReasoningEffort::Medium),
455 - ReasoningEffort::High => Ok(HarmonyReasoningEffort::High),
456 - ReasoningEffort::None
457 - | ReasoningEffort::Minimal
458 - | ReasoningEffort::XHigh
459 - | ReasoningEffort::Max => Err(Error::ChatTemplate(format!(
460 - "reasoning_effort={:?} is not supported by Harmony. Supported values are: low, medium, high.",
461 - reasoning_effort.as_str()
479 + match reasoning_effort.and_then(EffortValue::as_str) {
480 + Some("low") => Ok(HarmonyReasoningEffort::Low),
481 + Some("medium") => Ok(HarmonyReasoningEffort::Medium),
482 + Some("high") => Ok(HarmonyReasoningEffort::High),
483 + _ => Err(Error::InvalidReasoningEffort(format!(
484 + "reasoning_effort={} is not supported by Harmony. Supported values are: low, medium, high.",
485 + reasoning_effort.unwrap_or(&EffortValue::from("none"))
486 ))),
487 }
488 }
rust/src/chat/src/renderer/harmony/tests.rs
+41 -7
@@ -9,12 +9,11 @@ use thiserror_ext::AsReport as _;
9 use super::HarmonyChatRenderer;
10 use super::encoding::harmony_encoding;
11 use crate::ChatRenderer;
12 +use crate::EffortValue;
13 use crate::error::Error;
14 use crate::event::{AssistantContentBlock, AssistantToolCall};
15 use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
15 -use crate::request::{
16 - ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort,
17 -};
16 +use crate::request::{ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode};
17
18 const PINNED_DATE: &str = "2025-06-28";
19
@@ -22,7 +21,7 @@ fn fixture_request(input_name: &str) -> ChatRequest {
21 fixture_chat_request(
22 &fixture_path(input_name),
23 FixtureRequestOptions {
25 - enable_thinking: Some(false),
24 + enable_thinking: None,
25 no_generation_prompt_when_last_assistant: false,
26 },
27 )
@@ -36,7 +35,8 @@ fn fixture_path(name: &str) -> PathBuf {
35 }
36
37 fn test_renderer(use_system_instructions: bool) -> HarmonyChatRenderer {
39 - HarmonyChatRenderer::with_options(PINNED_DATE, use_system_instructions).unwrap()
38 + HarmonyChatRenderer::with_options(Default::default(), PINNED_DATE, use_system_instructions)
39 + .unwrap()
40 }
41
42 fn render_token_ids(request: &ChatRequest) -> Vec<u32> {
@@ -138,14 +138,48 @@ fn drops_stale_analysis_fixture() {
138 #[test]
139 fn rejects_invalid_reasoning_effort() {
140 let mut request = ChatRequest::for_test();
141 - request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
141 + request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
142
143 let error = test_renderer(false).render(&request).unwrap_err();
144
145 - expect![[r#"chat template error: reasoning_effort="none" is not supported by Harmony. Supported values are: low, medium, high."#]]
145 + expect![[r#"reasoning_effort="none" is not supported by Harmony. Supported values are: low, medium, high."#]]
146 .assert_eq(&error.to_report_string());
147 }
148
149 +#[test]
150 +fn normalized_reasoning_respects_harmony_capability_and_deployment_effort() {
151 + let renderer = HarmonyChatRenderer::with_options(
152 + [("reasoning_effort".to_string(), serde_json::json!("low"))].into(),
153 + PINNED_DATE,
154 + false,
155 + )
156 + .unwrap();
157 + let mut request = ChatRequest::for_test();
158 + request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
159 + request
160 + .chat_options
161 + .template_kwargs
162 + .insert("enable_thinking".into(), serde_json::json!(true));
163 + let rendered = renderer.render(&request).unwrap();
164 + let prompt = harmony_encoding()
165 + .unwrap()
166 + .tokenizer()
167 + .decode_utf8(rendered.prompt.into_token_ids().unwrap())
168 + .unwrap();
169 + assert!(prompt.contains("Reasoning: low"));
170 + assert_eq!(
171 + rendered.effective_template_kwargs["reasoning_effort"],
172 + "low"
173 + );
174 + assert_eq!(rendered.effective_template_kwargs["enable_thinking"], true);
175 +
176 + request
177 + .chat_options
178 + .template_kwargs
179 + .insert("thinking".into(), serde_json::json!(false));
180 + assert!(renderer.render(&request).is_err());
181 +}
182 +
183 #[test]
184 fn rejects_unknown_tool_response_id() {
185 let request = ChatRequest {
rust/src/chat/src/renderer/hf/mod.rs
+94 -4
@@ -16,7 +16,7 @@ use self::format::{
16 ChatTemplateContentFormat, ChatTemplateContentFormatOption as ContentFormatOption,
17 };
18 use self::template::{CompiledChatTemplate, TemplateContext};
19 -use super::{ChatRenderer, RenderedPrompt, effective_template_kwargs};
19 +use super::{ChatRenderer, RenderedPrompt};
20 use crate::error::Result;
21 use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
22 use crate::{
@@ -33,6 +33,31 @@ pub use template::{load_chat_template, resolve_chat_template};
33
34 pub use self::format::ChatTemplateContentFormatOption;
35
36 +/// Extract the effective chat-template kwargs visible to the renderer from the request,
37 +/// using the provided defaults as the base.
38 +fn effective_template_kwargs(
39 + default_template_kwargs: &HashMap<String, JsonValue>,
40 + request: &ChatRequest,
41 +) -> HashMap<String, JsonValue> {
42 + let mut kwargs = default_template_kwargs.clone();
43 + kwargs.extend(request.chat_options.template_kwargs.clone());
44 +
45 + if let Some(reasoning_effort) = &request.chat_options.reasoning_effort {
46 + kwargs.insert(
47 + "reasoning_effort".to_string(),
48 + serde_json::json!(reasoning_effort),
49 + );
50 + if !request.chat_options.template_kwargs.contains_key("enable_thinking") {
51 + kwargs.insert(
52 + "enable_thinking".to_string(),
53 + serde_json::json!(reasoning_effort.as_str() != Some("none")),
54 + );
55 + }
56 + }
57 +
58 + kwargs
59 +}
60 +
61 /// Template-visible placeholder tokens per supported modality.
62 ///
63 /// A `None` token means the loaded model does not support that modality, and
@@ -580,9 +605,10 @@ mod tests {
605 use vllm_text::backend::hf::{HfSpecialTokens, NamedSpecialToken};
606
607 use super::{ChatTemplateContentFormatOption, HfChatRenderer, MultimodalRenderInfo};
608 + use crate::EffortValue;
609 use crate::request::{
610 ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice,
585 - GenerationPromptMode, ReasoningEffort, ResolvedToolContext,
611 + GenerationPromptMode, ResolvedToolContext,
612 };
613 use crate::{AssistantContentBlock, ChatRenderer, Error, Result};
614
@@ -1149,7 +1175,7 @@ mod tests {
1175 #[test]
1176 fn chat_template_reasoning_effort_overrides_template_kwargs() {
1177 let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
1152 - request.chat_options.reasoning_effort = Some(ReasoningEffort::Max);
1178 + request.chat_options.reasoning_effort = Some(EffortValue::from("max"));
1179 request.chat_options.template_kwargs.insert(
1180 "reasoning_effort".to_string(),
1181 Value::String("low".to_string()),
@@ -1181,7 +1207,7 @@ mod tests {
1207 #[test]
1208 fn chat_template_reasoning_effort_preserves_request_enable_thinking() {
1209 let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);
1184 - request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
1210 + request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
1211 request
1212 .chat_options
1213 .template_kwargs
@@ -1207,6 +1233,70 @@ mod tests {
1233 );
1234 }
1235
1236 + #[test]
1237 + fn chat_template_preserves_typed_effort_values() {
1238 + for effort in [
1239 + serde_json::json!(37),
1240 + serde_json::json!(0.37),
1241 + serde_json::json!("custom"),
1242 + ] {
1243 + let mut request = sample_request(vec![ChatMessage::user("hello")]);
1244 + request.chat_options.reasoning_effort =
1245 + Some(serde_json::from_value(effort.clone()).unwrap());
1246 + request.chat_options.template_kwargs = [
1247 + ("reasoning_effort".to_string(), serde_json::json!("high")),
1248 + ("enable_thinking".to_string(), serde_json::json!(true)),
1249 + ]
1250 + .into();
1251 + let rendered = HfChatRenderer::new(
1252 + Some("{{ reasoning_effort|tojson }}|{{ enable_thinking }}".to_string()),
1253 + HashMap::new(),
1254 + ChatTemplateContentFormatOption::Auto,
1255 + )
1256 + .unwrap()
1257 + .render(&request)
1258 + .unwrap();
1259 + assert_eq!(rendered.prompt, Prompt::Text(format!("{effort}|True")));
1260 + assert_eq!(
1261 + rendered.effective_template_kwargs["reasoning_effort"],
1262 + effort
1263 + );
1264 + }
1265 + }
1266 +
1267 + #[test]
1268 + fn chat_template_keeps_native_control_conflicts_and_raw_effort_values() {
1269 + for effort in [
1270 + serde_json::json!(37),
1271 + serde_json::json!(null),
1272 + serde_json::json!("none"),
1273 + ] {
1274 + let mut request = sample_request(vec![ChatMessage::user("hello")]);
1275 + request.chat_options.template_kwargs = [
1276 + ("thinking".to_string(), serde_json::json!("custom-mode")),
1277 + ("enable_thinking".to_string(), serde_json::json!(true)),
1278 + ("reasoning_effort".to_string(), effort),
1279 + ]
1280 + .into();
1281 + let rendered = HfChatRenderer::new(
1282 + Some(
1283 + "{{ thinking }}|{{ enable_thinking }}|{{ reasoning_effort is none }}"
1284 + .to_string(),
1285 + ),
1286 + HashMap::new(),
1287 + ChatTemplateContentFormatOption::Auto,
1288 + )
1289 + .unwrap()
1290 + .render(&request)
1291 + .unwrap();
1292 + assert!(rendered.prompt.into_text().unwrap().starts_with("custom-mode|True|"));
1293 + assert_eq!(
1294 + rendered.effective_template_kwargs,
1295 + request.chat_options.template_kwargs
1296 + );
1297 + }
1298 + }
1299 +
1300 #[test]
1301 fn qwen3_template_omits_reasoning_for_historical_assistant_messages() {
1302 let request = sample_request(vec![
rust/src/chat/src/renderer/inkling/mod.rs
+47 -15
@@ -8,8 +8,10 @@ use thiserror_ext::AsReport as _;
8 use vllm_text::Prompt;
9 use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
10
11 -use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
11 +use super::{ChatRenderer, RenderedPrompt};
12 +use crate::EffortValue;
13 use crate::error::{Error, Result};
14 +use crate::reasoning::ReasoningControl;
15 use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool};
16 use crate::{AssistantContentBlock, AssistantToolCall};
17
@@ -35,6 +37,7 @@ const MAX_REASONING_EFFORT: f64 = 0.99;
37 pub struct InklingChatRenderer {
38 tokenizer: DynTokenizer,
39 special: InklingSpecialTokenIds,
40 + default_template_kwargs: HashMap<String, Value>,
41 }
42
43 #[derive(Debug, Clone, Copy)]
@@ -78,9 +81,16 @@ impl InklingSpecialTokenIds {
81 }
82
83 impl InklingChatRenderer {
81 - pub fn new(tokenizer: DynTokenizer) -> Result<Self> {
84 + pub fn new(
85 + tokenizer: DynTokenizer,
86 + default_template_kwargs: HashMap<String, Value>,
87 + ) -> Result<Self> {
88 let special = InklingSpecialTokenIds::resolve(tokenizer.as_ref())?;
83 - Ok(Self { tokenizer, special })
89 + Ok(Self {
90 + tokenizer,
91 + special,
92 + default_template_kwargs,
93 + })
94 }
95
96 fn write_text_tokens(&self, out: &mut Vec<u32>, text: &str) -> Result<()> {
@@ -150,11 +160,6 @@ impl InklingChatRenderer {
160 }
161
162 fn write_reasoning_effort(&self, out: &mut Vec<u32>, effort: f64) -> Result<()> {
153 - if !(0.0..=MAX_REASONING_EFFORT).contains(&effort) {
154 - return Err(Error::ChatTemplate(format!(
155 - "Inkling reasoning_effort must be in [0.0, 0.99], got {effort}"
156 - )));
157 - }
163 let formatted = format!("{effort:.2}");
164 let effort = formatted.trim_end_matches('0').trim_end_matches('.');
165 let effort = if matches!(effort, "0" | "-0") {
@@ -294,11 +299,18 @@ impl ChatRenderer for InklingChatRenderer {
299
300 let mut out = Vec::new();
301 let mut tool_call_id_to_name = HashMap::new();
297 - let effective_template_kwargs = request_template_kwargs(request);
302 + let reasoning = resolve_reasoning(ReasoningControl::resolve(
303 + request,
304 + &self.default_template_kwargs,
305 + )?)?;
306 + let effective_template_kwargs = reasoning.template_kwargs(request);
307 let tools = rendered_tools(request);
308 self.write_tool_declarations(&mut out, &tools)?;
300 - let mut reasoning_effort =
301 - resolve_reasoning_effort(effective_template_kwargs.get("reasoning_effort"));
309 + let mut reasoning_effort = if reasoning.is_enabled() {
310 + reasoning.effort().and_then(EffortValue::as_f64)
311 + } else {
312 + Some(0.0)
313 + };
314
315 for message in &request.messages {
316 if !matches!(
@@ -352,12 +364,33 @@ impl ChatRenderer for InklingChatRenderer {
364 }
365 }
366
355 -fn resolve_reasoning_effort(value: Option<&Value>) -> Option<f64> {
367 +fn resolve_reasoning(control: ReasoningControl) -> Result<ReasoningControl> {
368 + let control = control.fallback(ReasoningControl::enabled("high"));
369 + if !control.is_enabled() {
370 + return Ok(control);
371 + }
372 + let effort = resolve_reasoning_effort(control.effort());
373 + if let Some(effort) = effort {
374 + if !(0.0..=MAX_REASONING_EFFORT).contains(&effort) {
375 + return Err(Error::InvalidReasoningEffort(format!(
376 + "Inkling reasoning_effort must be in [0.0, 0.99], got {effort}"
377 + )));
378 + }
379 + if effort == 0.0 {
380 + return Ok(ReasoningControl::Disabled);
381 + }
382 + }
383 + Ok(ReasoningControl::Enabled {
384 + effort: effort.map(EffortValue::try_from).transpose()?,
385 + })
386 +}
387 +
388 +fn resolve_reasoning_effort(value: Option<&EffortValue>) -> Option<f64> {
389 let Some(value) = value else {
390 return Some(0.9);
391 };
392 match value {
360 - Value::String(name) => match name.as_str() {
393 + EffortValue::String(name) => match name.as_str() {
394 "none" => Some(0.0),
395 "minimal" => Some(0.1),
396 "low" => Some(0.2),
@@ -366,8 +399,7 @@ fn resolve_reasoning_effort(value: Option<&Value>) -> Option<f64> {
399 "xhigh" | "max" => Some(0.99),
400 _ => None,
401 },
369 - Value::Number(number) => number.as_f64(),
370 - _ => None,
402 + EffortValue::Number(number) => number.as_f64(),
403 }
404 }
405
rust/src/chat/src/renderer/inkling/tests.rs
+58 -22
@@ -9,6 +9,7 @@ use serde_json::json;
9 use thiserror_ext::AsReport;
10 use vllm_text::tokenizer::Tokenizer;
11
12 +use crate::EffortValue;
13 use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
14
15 use super::{
@@ -17,7 +18,7 @@ use super::{
18 MESSAGE_TOOL, MESSAGE_USER,
19 };
20 use crate::event::{AssistantContentBlock, AssistantToolCall};
20 -use crate::request::{ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort};
21 +use crate::request::{ChatMessage, ChatRequest, GenerationPromptMode};
22 use crate::{ChatRenderer, Error};
23
24 struct FixtureTokenizer;
@@ -87,7 +88,7 @@ impl Tokenizer for FixtureTokenizer {
88 }
89
90 fn renderer() -> InklingChatRenderer {
90 - InklingChatRenderer::new(Arc::new(FixtureTokenizer)).unwrap()
91 + InklingChatRenderer::new(Arc::new(FixtureTokenizer), Default::default()).unwrap()
92 }
93
94 fn render_token_ids(request: &ChatRequest) -> Vec<u32> {
@@ -105,7 +106,7 @@ fn fixture_request(name: &str) -> ChatRequest {
106
107 fn inkling_fixture_options() -> FixtureRequestOptions {
108 FixtureRequestOptions {
108 - enable_thinking: Some(false),
109 + enable_thinking: None,
110 no_generation_prompt_when_last_assistant: false,
111 }
112 }
@@ -206,13 +207,13 @@ fn renders_request_and_developer_tools_as_tool_declare() {
207 #[test]
208 fn renders_named_reasoning_effort_after_tool_declarations() {
209 for (effort, expected) in [
209 - (ReasoningEffort::None, "0.0"),
210 - (ReasoningEffort::Minimal, "0.1"),
211 - (ReasoningEffort::Low, "0.2"),
212 - (ReasoningEffort::Medium, "0.7"),
213 - (ReasoningEffort::High, "0.9"),
214 - (ReasoningEffort::XHigh, "0.99"),
215 - (ReasoningEffort::Max, "0.99"),
210 + (EffortValue::from("none"), "0.0"),
211 + (EffortValue::from("minimal"), "0.1"),
212 + (EffortValue::from("low"), "0.2"),
213 + (EffortValue::from("medium"), "0.7"),
214 + (EffortValue::from("high"), "0.9"),
215 + (EffortValue::from("xhigh"), "0.99"),
216 + (EffortValue::from("max"), "0.99"),
217 ] {
218 let mut request = fixture_request("tool_declare_input.json");
219 request.chat_options.reasoning_effort = Some(effort);
@@ -284,6 +285,43 @@ fn defaults_reasoning_effort_to_high() {
285 );
286 }
287
288 +#[test]
289 +fn normalized_mode_and_scalar_are_reflected_in_prompt_and_kwargs() {
290 + for (kwargs, expected, enabled) in [
291 + (
292 + json!({"enable_thinking": true, "reasoning_effort": "none"}),
293 + "0.9",
294 + true,
295 + ),
296 + (
297 + json!({"thinking": false, "reasoning_effort": 10}),
298 + "0.0",
299 + false,
300 + ),
301 + (json!({"reasoning_effort": 0}), "0.0", false),
302 + (json!({"reasoning_effort": null}), "0.9", true),
303 + (json!({"reasoning_effort": 0.37}), "0.37", true),
304 + ] {
305 + let mut request = ChatRequest::for_test();
306 + request.chat_options.template_kwargs = serde_json::from_value(kwargs).unwrap();
307 + let rendered = renderer().render(&request).unwrap();
308 + let prompt = render_symbolic_tokens(&rendered.prompt.into_token_ids().unwrap());
309 + assert!(prompt.contains(&format!("Thinking effort level: {expected}<|end_message|>")));
310 + assert_eq!(
311 + rendered.effective_template_kwargs["enable_thinking"],
312 + enabled
313 + );
314 + assert_eq!(
315 + rendered.effective_template_kwargs["reasoning_effort"],
316 + if enabled {
317 + json!(expected.parse::<f64>().unwrap())
318 + } else {
319 + json!("none")
320 + }
321 + );
322 + }
323 +}
324 +
325 #[test]
326 fn renders_numeric_reasoning_effort_template_kwarg() {
327 let mut request = ChatRequest::for_test();
@@ -301,19 +339,17 @@ fn renders_numeric_reasoning_effort_template_kwarg() {
339 }
340
341 #[test]
304 -fn ignores_unsupported_reasoning_effort_values() {
305 - for value in [json!(true), json!("invalid"), json!(null)] {
306 - let mut request = ChatRequest::for_test();
307 - request
308 - .chat_options
309 - .template_kwargs
310 - .insert("reasoning_effort".to_string(), value);
342 +fn ignores_unsupported_reasoning_effort_names() {
343 + let mut request = ChatRequest::for_test();
344 + request
345 + .chat_options
346 + .template_kwargs
347 + .insert("reasoning_effort".to_string(), json!("invalid"));
348
312 - assert_eq!(
313 - render_symbolic_tokens(&render_token_ids(&request)),
314 - "<|message_user|><|content_text|>test<|end_message|><|message_model|>"
315 - );
316 - }
349 + assert_eq!(
350 + render_symbolic_tokens(&render_token_ids(&request)),
351 + "<|message_user|><|content_text|>test<|end_message|><|message_model|>"
352 + );
353 }
354
355 #[test]
rust/src/chat/src/renderer/kimi_k3/encoding.rs
+20 -49
@@ -11,7 +11,9 @@ use serde_json::{Map, Value, json};
11 use vllm_tokenizer::Tokenizer;
12
13 use super::super::MediaPartSource;
14 +use crate::EffortValue;
15 use crate::error::{Error, Result};
16 +use crate::reasoning::ReasoningControl;
17 use crate::request::{
18 ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice,
19 };
@@ -76,9 +78,10 @@ impl<'a> K3TokenWriter<'a> {
78 pub(super) fn render_request_with_media_order(
79 request: &ChatRequest,
80 tokenizer: &dyn Tokenizer,
81 + reasoning: &ReasoningControl,
82 ) -> Result<(Vec<u32>, Vec<MediaPartSource>)> {
80 - let thinking = thinking_enabled(request)?;
81 - let thinking_effort = thinking.then(|| thinking_effort(request)).transpose()?;
83 + let thinking = reasoning.is_enabled();
84 + let thinking_effort = reasoning.effort().and_then(EffortValue::as_str);
85 let tools = request_tools(request);
86 let mut out = K3TokenWriter::new(tokenizer);
87
@@ -244,58 +247,26 @@ fn request_tools(request: &ChatRequest) -> &[ChatTool] {
247 request.initial_tools()
248 }
249
247 -fn thinking_enabled(request: &ChatRequest) -> Result<bool> {
248 - if let Some(thinking) = request.parse_template_bool("thinking")? {
249 - return Ok(thinking);
250 - }
251 - if let Some(enable_thinking) = request.parse_template_bool("enable_thinking")? {
252 - return Ok(enable_thinking);
253 - }
254 - if let Some(reasoning_effort) = request.chat_options.reasoning_effort {
255 - return Ok(reasoning_effort != crate::request::ReasoningEffort::None);
256 - }
257 - Ok(request
258 - .chat_options
259 - .template_kwargs
260 - .get("reasoning_effort")
261 - .and_then(Value::as_str)
262 - != Some("none"))
263 -}
264 -
265 -fn thinking_effort(request: &ChatRequest) -> Result<String> {
266 - let effort = if let Some(value) = request.chat_options.template_kwargs.get("thinking_effort") {
267 - value.as_str().ok_or_else(|| {
268 - Error::ChatTemplate(format!(
269 - "template kwarg `thinking_effort` must be a string, got {value}"
270 - ))
271 - })?
272 - } else if let Some(effort) = request.chat_options.reasoning_effort {
273 - if effort == crate::request::ReasoningEffort::None {
274 - DEFAULT_THINKING_EFFORT
275 - } else {
276 - effort.as_str()
277 - }
278 - } else if let Some(value) = request.chat_options.template_kwargs.get("reasoning_effort") {
250 +/// Resolve standard reasoning controls and validate K3's supported effort grades.
251 +pub(super) fn resolve_reasoning(
252 + request: &ChatRequest,
253 + defaults: &HashMap<String, Value>,
254 +) -> Result<ReasoningControl> {
255 + let reasoning = ReasoningControl::resolve(request, defaults)?
256 + .fallback(ReasoningControl::enabled(DEFAULT_THINKING_EFFORT));
257 + if let Some(value) = reasoning.effort() {
258 let effort = value.as_str().ok_or_else(|| {
280 - Error::ChatTemplate(format!(
281 - "template kwarg `reasoning_effort` must be a string, got {value}"
259 + Error::InvalidReasoningEffort(format!(
260 + "Kimi K3 reasoning_effort must be a string, got {value}"
261 ))
262 })?;
284 - if effort == "none" {
285 - DEFAULT_THINKING_EFFORT
286 - } else {
287 - effort
263 + if !VALID_THINKING_EFFORTS.contains(&effort) {
264 + return Err(Error::InvalidReasoningEffort(format!(
265 + "unsupported reasoning_effort={effort:?}; supported values are `low`, `high`, and `max`"
266 + )));
267 }
289 - } else {
290 - DEFAULT_THINKING_EFFORT
291 - };
292 -
293 - if !VALID_THINKING_EFFORTS.contains(&effort) {
294 - return Err(Error::ChatTemplate(format!(
295 - "unsupported thinking_effort={effort:?}; supported values are `low`, `high`, and `max`"
296 - )));
268 }
298 - Ok(effort.to_string())
269 + Ok(reasoning)
270 }
271
272 fn content_is_empty(content: &ChatContent) -> bool {
rust/src/chat/src/renderer/kimi_k3/mod.rs
+26 -6
@@ -7,10 +7,13 @@ mod encoding;
7 #[cfg(test)]
8 mod tests;
9
10 +use std::collections::HashMap;
11 +
12 +use serde_json::Value;
13 use vllm_text::Prompt;
14 use vllm_text::tokenizer::DynTokenizer;
15
13 -use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};
16 +use super::{ChatRenderer, RenderedPrompt};
17 use crate::Result;
18 use crate::request::ChatRequest;
19
@@ -18,25 +21,42 @@ use crate::request::ChatRequest;
21 #[derive(Clone)]
22 pub struct KimiK3ChatRenderer {
23 tokenizer: DynTokenizer,
24 + default_template_kwargs: HashMap<String, Value>,
25 }
26
27 impl KimiK3ChatRenderer {
28 /// Create a Kimi K3 renderer.
25 - pub fn new(tokenizer: DynTokenizer) -> Self {
26 - Self { tokenizer }
29 + pub fn new(tokenizer: DynTokenizer, default_template_kwargs: HashMap<String, Value>) -> Self {
30 + Self {
31 + tokenizer,
32 + default_template_kwargs,
33 + }
34 }
35 }
36
37 impl ChatRenderer for KimiK3ChatRenderer {
38 fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
39 request.validate()?;
33 - let (token_ids, media_order) =
34 - encoding::render_request_with_media_order(request, self.tokenizer.as_ref())?;
40 + let reasoning = encoding::resolve_reasoning(request, &self.default_template_kwargs)?;
41 + let mut effective_template_kwargs = reasoning.template_kwargs(request);
42 + // Export K3's native effort key from the resolved standard controls.
43 + // Input `thinking_effort` is ignored; clear it when reasoning is disabled.
44 + if let Some(effort) = reasoning.effort() {
45 + effective_template_kwargs
46 + .insert("thinking_effort".to_string(), serde_json::json!(effort));
47 + } else {
48 + effective_template_kwargs.remove("thinking_effort");
49 + }
50 + let (token_ids, media_order) = encoding::render_request_with_media_order(
51 + request,
52 + self.tokenizer.as_ref(),
53 + &reasoning,
54 + )?;
55
56 Ok(RenderedPrompt {
57 prompt: Prompt::TokenIds(token_ids),
58 media_order: Some(media_order),
39 - effective_template_kwargs: request_template_kwargs(request),
59 + effective_template_kwargs,
60 })
61 }
62 }
rust/src/chat/src/renderer/kimi_k3/tests.rs
+80 -19
@@ -15,11 +15,10 @@ use vllm_tokenizer::test_utils::TestTokenizer;
15
16 use super::KimiK3ChatRenderer;
17 use crate::ChatRenderer;
18 +use crate::EffortValue;
19 use crate::renderer::kimi_k3::encoding::{CLOSE, END_OF_MSG, IMAGE_PLACEHOLDER, OPEN, SEP};
20 use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
20 -use crate::request::{
21 - ChatContentPart, ChatMessage, ChatTool, GenerationPromptMode, ReasoningEffort,
22 -};
21 +use crate::request::{ChatContentPart, ChatMessage, ChatTool, GenerationPromptMode};
22 use crate::{AssistantContentBlock, AssistantToolCall};
23
24 const OPEN_ID: u32 = 256;
@@ -37,8 +36,70 @@ fn test_tokenizer() -> TestTokenizer {
36 .with_special_token(IMAGE_PLACEHOLDER, MEDIA_ID)
37 }
38
39 +#[test]
40 +fn standard_effort_controls_prompt_and_exported_native_effort() {
41 + let tokenizer = Arc::new(test_tokenizer());
42 + for (kwargs, typed, defaults, effort) in [
43 + (json!({}), None, json!({}), "max"),
44 + (
45 + json!({}),
46 + Some(EffortValue::from("low")),
47 + json!({"reasoning_effort": "high"}),
48 + "low",
49 + ),
50 + (
51 + json!({"enable_thinking": true, "reasoning_effort": "none"}),
52 + None,
53 + json!({"reasoning_effort": "low"}),
54 + "low",
55 + ),
56 + (
57 + json!({}),
58 + Some(EffortValue::from("none")),
59 + json!({"reasoning_effort": "low"}),
60 + "none",
61 + ),
62 + (
63 + json!({"thinking": true}),
64 + Some(EffortValue::from("none")),
65 + json!({}),
66 + "max",
67 + ),
68 + ] {
69 + let renderer =
70 + KimiK3ChatRenderer::new(tokenizer.clone(), serde_json::from_value(defaults).unwrap());
71 + let mut request = crate::ChatRequest::for_test();
72 + request.chat_options.reasoning_effort = typed;
73 + request.chat_options.template_kwargs = serde_json::from_value(kwargs).unwrap();
74 + let rendered = renderer.render(&request).unwrap();
75 + let prompt = tokenizer.decode(&rendered.prompt.into_token_ids().unwrap(), false).unwrap();
76 + let enabled = effort != "none";
77 + assert_eq!(prompt.ends_with("<|open|>think<|sep|>"), enabled);
78 + assert_eq!(
79 + rendered.effective_template_kwargs["reasoning_effort"],
80 + effort
81 + );
82 + assert_eq!(
83 + rendered.effective_template_kwargs["enable_thinking"],
84 + enabled
85 + );
86 + if enabled {
87 + assert!(prompt.contains(&format!("thinking_effort={effort}")));
88 + assert_eq!(
89 + rendered.effective_template_kwargs["thinking_effort"],
90 + effort
91 + );
92 + } else {
93 + assert!(!rendered.effective_template_kwargs.contains_key("thinking_effort"));
94 + }
95 + }
96 +}
97 +
98 fn render_token_ids(request: &crate::request::ChatRequest, tokenizer: DynTokenizer) -> Vec<u32> {
41 - let prompt = KimiK3ChatRenderer::new(tokenizer).render(request).unwrap().prompt;
99 + let prompt = KimiK3ChatRenderer::new(tokenizer, Default::default())
100 + .render(request)
101 + .unwrap()
102 + .prompt;
103 let Prompt::TokenIds(token_ids) = prompt else {
104 panic!("kimi k3 renderer should return token IDs")
105 };
@@ -67,7 +128,11 @@ fn kimi_k3_fixture_options() -> FixtureRequestOptions {
128
129 fn assert_golden(name: &str) {
130 let input_name = format!("{name}_input.json");
70 - let request = fixture_chat_request(&fixture_path(&input_name), kimi_k3_fixture_options());
131 + let mut request = fixture_chat_request(&fixture_path(&input_name), kimi_k3_fixture_options());
132 + // Reference fixtures use encoder-native keys; Rust accepts standard effort input.
133 + if let Some(effort) = request.chat_options.template_kwargs.remove("thinking_effort") {
134 + request.chat_options.reasoning_effort = Some(EffortValue::try_from(effort).unwrap());
135 + }
136 let rendered = render_request(&request);
137 expect_file![format!("fixtures/{name}_output.txt")].assert_eq(&rendered);
138 }
@@ -310,7 +375,7 @@ fn media_order_follows_reordered_tool_results() {
375 ),
376 ];
377 request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
313 - let renderer = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()));
378 + let renderer = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()), Default::default());
379
380 let rendered = renderer.render(&request).unwrap();
381 let media_order = rendered.media_order.unwrap();
@@ -371,19 +436,18 @@ fn translates_standard_thinking_kwargs() {
436 }
437
438 #[test]
374 -fn native_k3_kwargs_take_precedence() {
439 +fn thinking_alias_takes_precedence() {
440 let mut request = crate::request::ChatRequest::for_test();
441 request.chat_options.template_kwargs.extend([
442 ("thinking".to_string(), json!(true)),
443 ("enable_thinking".to_string(), json!(false)),
379 - ("thinking_effort".to_string(), json!("low")),
444 ("reasoning_effort".to_string(), json!("high")),
445 ]);
446
447 let rendered = render_request(&request);
448
385 - assert!(rendered.contains("thinking_effort=low"));
386 - assert!(!rendered.contains("thinking_effort=high"));
449 + assert!(rendered.contains("thinking_effort=high"));
450 + assert!(rendered.ends_with("<|open|>think<|sep|>"));
451 }
452
453 #[test]
@@ -431,7 +495,7 @@ fn enable_thinking_true_overrides_standard_none() {
495 #[test]
496 fn typed_none_disables_thinking() {
497 let mut request = crate::request::ChatRequest::for_test();
434 - request.chat_options.reasoning_effort = Some(ReasoningEffort::None);
498 + request.chat_options.reasoning_effort = Some(EffortValue::from("none"));
499
500 let rendered = render_request(&request);
501
@@ -440,20 +504,17 @@ fn typed_none_disables_thinking() {
504 }
505
506 #[test]
443 -fn rejects_removed_medium_thinking_effort() {
507 +fn rejects_unsupported_standard_reasoning_effort() {
508 let mut request = crate::request::ChatRequest::for_test();
445 - request
446 - .chat_options
447 - .template_kwargs
448 - .insert("thinking_effort".to_string(), json!("medium"));
509 + request.chat_options.reasoning_effort = Some(EffortValue::from("medium"));
510
450 - let error = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()))
511 + let error = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()), Default::default())
512 .render(&request)
513 .unwrap_err();
514
515 expect![[r#"
455 - ChatTemplate(
456 - "unsupported thinking_effort=\"medium\"; supported values are `low`, `high`, and `max`",
516 + InvalidReasoningEffort(
517 + "unsupported reasoning_effort=\"medium\"; supported values are `low`, `high`, and `max`",
518 )
519 "#]]
520 .assert_debug_eq(&error);
rust/src/chat/src/renderer/mod.rs
+2 -32
@@ -4,11 +4,11 @@
4 use std::collections::HashMap;
5 use std::sync::Arc;
6
7 -use serde_json::{Value, json};
7 +use serde_json::Value;
8 use vllm_text::Prompt;
9
10 use crate::error::Result;
11 -use crate::request::{ChatRequest, ReasoningEffort};
11 +use crate::request::ChatRequest;
12
13 mod deepseek;
14 pub mod deepseek_v32;
@@ -62,33 +62,3 @@ pub trait ChatRenderer: Send + Sync {
62
63 /// Shared trait-object form of [`ChatRenderer`].
64 pub type DynChatRenderer = Arc<dyn ChatRenderer>;
65 -
66 -/// Extract the effective chat-template kwargs visible to the renderer from the request,
67 -/// using the provided defaults as the base.
68 -pub(crate) fn effective_template_kwargs(
69 - default_template_kwargs: &HashMap<String, Value>,
70 - request: &ChatRequest,
71 -) -> HashMap<String, Value> {
72 - let mut kwargs = default_template_kwargs.clone();
73 - kwargs.extend(request.chat_options.template_kwargs.clone());
74 -
75 - if let Some(reasoning_effort) = request.chat_options.reasoning_effort {
76 - kwargs.insert(
77 - "reasoning_effort".to_string(),
78 - Value::String(reasoning_effort.as_str().to_string()),
79 - );
80 - if !request.chat_options.template_kwargs.contains_key("enable_thinking") {
81 - kwargs.insert(
82 - "enable_thinking".to_string(),
83 - json!(reasoning_effort != ReasoningEffort::None),
84 - );
85 - }
86 - }
87 -
88 - kwargs
89 -}
90 -
91 -/// Extract the effective chat-template kwargs visible to the renderer from the request.
92 -pub(crate) fn request_template_kwargs(request: &ChatRequest) -> HashMap<String, Value> {
93 - effective_template_kwargs(&HashMap::new(), request)
94 -}
rust/src/chat/src/renderer/test_utils.rs
+3 -2
@@ -8,10 +8,11 @@ use std::path::Path;
8 use serde::Deserialize;
9 use serde_json::Value;
10
11 +use crate::EffortValue;
12 use crate::event::{AssistantContentBlock, AssistantToolCall};
13 use crate::request::{
14 ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice,
14 - GenerationPromptMode, ReasoningEffort, ResolvedToolContext,
15 + GenerationPromptMode, ResolvedToolContext,
16 };
17
18 /// Options for constructing a [`ChatRequest`] from a fixture file.
@@ -47,7 +48,7 @@ pub(crate) struct FixtureRequest {
48 tools: Vec<FixtureTool>,
49 messages: Vec<FixtureMessage>,
50 add_generation_prompt: Option<bool>,
50 - reasoning_effort: Option<ReasoningEffort>,
51 + reasoning_effort: Option<EffortValue>,
52 /// Standard response format passed to model-specific renderers.
53 #[serde(default)]
54 response_format: Option<Value>,
rust/src/chat/src/request.rs
+6 -106
@@ -11,9 +11,9 @@ pub use vllm_parser::tool::Tool as ChatTool;
11 use vllm_text::TextDecodeOptions;
12 pub use vllm_text::{PromptTruncation, SamplingParams};
13
14 -use crate::AssistantMessageExt;
14 use crate::error::{Error, Result};
15 use crate::event::{AssistantContentBlock, AssistantMessage};
16 +use crate::{AssistantMessageExt, EffortValue};
17
18 /// Role label for one text-only chat message.
19 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -362,33 +362,6 @@ pub enum GenerationPromptMode {
362 NoGenerationPrompt,
363 }
364
365 -/// Effort level for reasoning models.
366 -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
367 -#[serde(rename_all = "lowercase")]
368 -pub enum ReasoningEffort {
369 - None,
370 - Minimal,
371 - Low,
372 - Medium,
373 - High,
374 - XHigh,
375 - Max,
376 -}
377 -
378 -impl ReasoningEffort {
379 - pub fn as_str(self) -> &'static str {
380 - match self {
381 - Self::None => "none",
382 - Self::Minimal => "minimal",
383 - Self::Low => "low",
384 - Self::Medium => "medium",
385 - Self::High => "high",
386 - Self::XHigh => "xhigh",
387 - Self::Max => "max",
388 - }
389 - }
390 -}
391 -
365 /// Chat-template-related request options.
366 ///
367 /// These are the small subset of chat controls that currently affect prompt
@@ -404,8 +377,10 @@ pub struct ChatOptions {
377 /// used instead of the model's default chat template.
378 pub chat_template: Option<String>,
379
407 - /// Effort level exposed to chat templates for reasoning models.
408 - pub reasoning_effort: Option<ReasoningEffort>,
380 + /// Model-specific reasoning effort, typically `none`, `minimal`, `low`,
381 + /// `medium`, `high`, `xhigh`, or `max`. Supported names and numeric ranges
382 + /// are validated by the selected renderer or HF template.
383 + pub reasoning_effort: Option<EffortValue>,
384
385 /// Standard response format available to model-specific renderers.
386 #[serde(default)]
@@ -673,29 +648,6 @@ impl ChatRequest {
648 self.tool_context.parsing_enabled()
649 }
650
676 - /// Return the request-level thinking toggle when explicitly requested.
677 - ///
678 - /// We currently accept the two request kwargs `thinking` and
679 - /// `enable_thinking`. Both must be booleans when present. If both are
680 - /// present, they must have the same value. If neither key is provided,
681 - /// return `None`.
682 - pub(crate) fn enable_thinking(&self) -> Result<Option<bool>> {
683 - let thinking = self.parse_template_bool("thinking")?;
684 - let enable_thinking = self.parse_template_bool("enable_thinking")?;
685 -
686 - match (thinking, enable_thinking) {
687 - (None, None) => Ok(None),
688 - (Some(thinking), Some(enable_thinking)) if thinking != enable_thinking => {
689 - Err(Error::ChatTemplate(
690 - "template kwargs `thinking` and `enable_thinking` must match when both are set"
691 - .to_string(),
692 - ))
693 - }
694 - (Some(thinking), _) => Ok(Some(thinking)),
695 - (None, Some(enable_thinking)) => Ok(Some(enable_thinking)),
696 - }
697 - }
698 -
651 pub(crate) fn parse_template_bool(&self, key: &str) -> Result<Option<bool>> {
652 match self.chat_options.template_kwargs.get(key) {
653 None => Ok(None),
@@ -726,7 +678,7 @@ mod tests {
678 use serde_json::{json, to_value};
679
680 use super::{
729 - ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice,
681 + ChatContent, ChatContentPart, ChatMessage, ChatRole, ChatTool, ChatToolChoice,
682 ResolvedToolContext,
683 };
684 use crate::Error;
@@ -919,56 +871,4 @@ mod tests {
871 Error::ToolChoiceFunctionNotFound { name } if name == "missing"
872 ));
873 }
922 -
923 - #[test]
924 - fn enable_thinking_is_none_when_no_kwargs_are_present() {
925 - let request = ChatRequest::for_test();
926 - assert_eq!(request.enable_thinking().unwrap(), None);
927 - }
928 -
929 - #[test]
930 - fn enable_thinking_accepts_matching_duplicate_kwargs() {
931 - let mut request = ChatRequest::for_test();
932 - request.chat_options.template_kwargs.insert("thinking".to_string(), json!(true));
933 - request
934 - .chat_options
935 - .template_kwargs
936 - .insert("enable_thinking".to_string(), json!(true));
937 -
938 - assert_eq!(request.enable_thinking().unwrap(), Some(true));
939 - }
940 -
941 - #[test]
942 - fn enable_thinking_rejects_non_boolean_kwargs() {
943 - let mut request = ChatRequest::for_test();
944 - request
945 - .chat_options
946 - .template_kwargs
947 - .insert("thinking".to_string(), json!("yes"));
948 -
949 - assert!(matches!(
950 - request.enable_thinking(),
951 - Err(Error::ChatTemplate(message))
952 - if message.contains("`thinking` must be a boolean")
953 - ));
954 - }
955 -
956 - #[test]
957 - fn enable_thinking_rejects_conflicting_duplicate_kwargs() {
958 - let mut request = ChatRequest::for_test();
959 - request
960 - .chat_options
961 - .template_kwargs
962 - .insert("thinking".to_string(), json!(false));
963 - request
964 - .chat_options
965 - .template_kwargs
966 - .insert("enable_thinking".to_string(), json!(true));
967 -
968 - assert!(matches!(
969 - request.enable_thinking(),
970 - Err(Error::ChatTemplate(message))
971 - if message.contains("`thinking` and `enable_thinking` must match")
972 - ));
973 - }
874 }
rust/src/server/src/error.rs
+16 -10
@@ -175,16 +175,22 @@ mod tests {
175 }
176
177 #[test]
178 - fn invalid_reasoning_effort_maps_to_invalid_request() {
179 - let error = vllm_chat::Error::InvalidReasoningEffort(
180 - "DeepSeek V4.1 reasoning_effort must be within [1, 100]".to_string(),
181 - );
182 - let api_error = chat_submit_error("failed to submit chat request", error);
183 - assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
184 - assert_eq!(
185 - api_error.to_error_response().error.error_type,
186 - "invalid_request_error"
187 - );
178 + fn invalid_reasoning_parameters_map_to_invalid_request() {
179 + for error in [
180 + vllm_chat::Error::InvalidReasoningEffort(
181 + "DeepSeek V4.1 reasoning_effort must be within [1, 100]".to_string(),
182 + ),
183 + vllm_chat::Error::InvalidReasoningControl {
184 + message: "template kwarg `thinking` must be a boolean".to_string(),
185 + },
186 + ] {
187 + let api_error = chat_submit_error("failed to submit chat request", error);
188 + assert_eq!(api_error.status_code(), StatusCode::BAD_REQUEST);
189 + assert_eq!(
190 + api_error.to_error_response().error.error_type,
191 + "invalid_request_error"
192 + );
193 + }
194 }
195
196 #[test]
rust/src/server/src/routes/openai/chat_completions/convert.rs
+46 -2
@@ -180,7 +180,7 @@ pub(super) fn prepare_chat_request(
180 chat_options: ChatOptions {
181 generation_prompt_mode,
182 chat_template: request.chat_template,
183 - reasoning_effort: request.reasoning_effort,
183 + reasoning_effort: request.reasoning_effort.map(|effort| effort.as_str().into()),
184 response_format,
185 template_kwargs,
186 },
@@ -488,6 +488,50 @@ mod tests {
488 }
489 }
490
491 + #[test]
492 + fn chat_http_reasoning_effort_preserves_omission_none_and_kwargs() {
493 + for effort in [None, Some(json!(null))].into_iter().chain(
494 + ["none", "minimal", "low", "medium", "high", "xhigh", "max"]
495 + .map(|name| Some(json!(name))),
496 + ) {
497 + let kwargs = json!({"reasoning_effort": 37, "thinking": true});
498 + let mut value = json!({
499 + "messages": [{"role": "user", "content": "hello"}],
500 + "chat_template_kwargs": kwargs,
501 + });
502 + if let Some(effort) = &effort {
503 + value["reasoning_effort"] = effort.clone();
504 + }
505 + let request: ChatCompletionRequest = serde_json::from_value(value).unwrap();
506 + let prepared = prepare_chat_request(
507 + request,
508 + &served(&["test-model"]),
509 + request_context(&HeaderMap::new(), None),
510 + )
511 + .unwrap();
512 + let options = prepared.chat_request.chat_options;
513 + assert_eq!(
514 + serde_json::to_value(options.reasoning_effort).unwrap(),
515 + effort.unwrap_or(serde_json::Value::Null)
516 + );
517 + assert_eq!(
518 + serde_json::to_value(options.template_kwargs).unwrap(),
519 + kwargs
520 + );
521 + }
522 + }
523 +
524 + #[test]
525 + fn chat_http_reasoning_effort_rejects_model_extensions_at_top_level() {
526 + for effort in [json!(37), json!("custom")] {
527 + let request = json!({
528 + "messages": [{"role": "user", "content": "hello"}],
529 + "reasoning_effort": effort,
530 + });
531 + assert!(serde_json::from_value::<ChatCompletionRequest>(request).is_err());
532 + }
533 + }
534 +
535 #[test]
536 fn chat_http_request_defaults_missing_or_null_model() {
537 for model in [None, Some(serde_json::Value::Null)] {
@@ -747,7 +791,7 @@ mod tests {
791 );
792
793 let tokenizer = Arc::new(TestTokenizer::new());
750 - let prompt = KimiK3ChatRenderer::new(tokenizer.clone())
794 + let prompt = KimiK3ChatRenderer::new(tokenizer.clone(), Default::default())
795 .render(&prepared.chat_request)
796 .expect("Kimi K3 rendering succeeds")
797 .prompt;
rust/src/server/src/routes/openai/chat_completions/types.rs
+4 -4
@@ -9,15 +9,15 @@ use serde::{Deserialize, Serialize};
9 use serde_json::Value;
10 use serde_with::SerializeDisplay;
11 use validator::Validate;
12 -use vllm_chat::ReasoningEffort;
12 use vllm_engine_core_client::protocol::sampling::RepetitionDetectionParams;
13 use vllm_text::TruncationSide;
14
15 use crate::routes::openai::utils::structured_outputs::ResponseFormat;
16 use crate::routes::openai::utils::types::{
18 - ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, StreamOptions, StreamResponseEnvelope,
19 - StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, Usage, default_true,
20 - deserialize_request_top_k, validate_messages, validate_stop, validate_top_p_value,
17 + ChatLogProbs, ChatMessage, Normalizable, PromptLogprobs, ReasoningEffort, StreamOptions,
18 + StreamResponseEnvelope, StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, Usage,
19 + default_true, deserialize_request_top_k, validate_messages, validate_stop,
20 + validate_top_p_value,
21 };
22
23 /// vLLM-compatible request type for the Chat Completions API.
rust/src/server/src/routes/openai/chat_completions/validate.rs
+1 -1
@@ -125,8 +125,8 @@ fn validate_function_tools(tools: &[Tool], param: &'static str) -> Result<(), Ap
125 mod tests {
126 use std::collections::HashMap;
127
128 + use crate::routes::openai::utils::types::ReasoningEffort;
129 use serde_json::json;
129 - use vllm_chat::ReasoningEffort;
130
131 use super::validate_request_compat;
132 use crate::routes::openai::chat_completions::types::ChatCompletionRequest;
rust/src/server/src/routes/openai/utils/types.rs
+30
@@ -39,6 +39,36 @@ where
39 }
40 }
41
42 +/// Effort level for reasoning models.
43 +///
44 +/// Fixed OpenAI HTTP request grades. Request conversion maps these names to
45 +/// the model-independent `vllm_chat::EffortValue` before renderer validation.
46 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
47 +#[serde(rename_all = "lowercase")]
48 +pub enum ReasoningEffort {
49 + None,
50 + Minimal,
51 + Low,
52 + Medium,
53 + High,
54 + XHigh,
55 + Max,
56 +}
57 +
58 +impl ReasoningEffort {
59 + pub fn as_str(self) -> &'static str {
60 + match self {
61 + Self::None => "none",
62 + Self::Minimal => "minimal",
63 + Self::Low => "low",
64 + Self::Medium => "medium",
65 + Self::High => "high",
66 + Self::XHigh => "xhigh",
67 + Self::Max => "max",
68 + }
69 + }
70 +}
71 +
72 // ============================================================================
73 // String/Array Utilities
74 // ============================================================================