@tduncan / Nvidia-Optimization-Stack-VLLM / commits / ce9f64020b

[Rust Frontend] Pass effective `reasoning_parser_kwargs` for structured output (#46360)

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Jun 24, 2026 at 11:13 UTC ce9f64020ba94b2fb1d318f15cfb579bf0500293

19 files changed +165 -34

rust/src/chat/src/lib.rs

+10 -1

@@ -50,7 +50,7 @@ mod request;

50 mod stream;

52 use vllm_engine_core_client::EngineCoreClient;

53 -use vllm_engine_core_client::protocol::ModelDtype;

53 +use vllm_engine_core_client::protocol::{ModelDtype, ReasoningParserKwargs};

54 use vllm_llm::Llm;

55 use vllm_text::{Prompt, TextLlm, TextRequest};

@@ -179,6 +179,14 @@ impl ChatLlm {

179 },

180 )?;

181 let rendered = self.backend.chat_renderer().render(&request)?;

182 + let reasoning_parser_kwargs =

183 + request

184 + .sampling_params

185 + .structured_outputs

186 + .is_some()

187 + .then(|| ReasoningParserKwargs {

188 + chat_template_kwargs: rendered.effective_template_kwargs.clone(),

189 + });

190

191 let (prompt, mm_features) = multimodal::finalize_rendered_prompt(

192 &request,

@@ -199,6 +207,7 @@ impl ChatLlm {

207 cache_salt: request.cache_salt,

208 add_special_tokens: request.add_special_tokens,

209 data_parallel_rank: request.data_parallel_rank,

210 + reasoning_parser_kwargs,

211 lora_request: request.lora_request,

212 };

213 let decoded_stream = self.text.generate(text_request).await?.map_err(Error::from).boxed();

rust/src/chat/src/renderer/deepseek_v32/mod.rs

+2 -1

@@ -2,7 +2,7 @@ mod encoding;

3 use vllm_text::Prompt;

5 -use super::{ChatRenderer, RenderedPrompt};

5 +use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};

6 use crate::Result;

7 use crate::request::ChatRequest;

@@ -23,6 +23,7 @@ impl ChatRenderer for DeepSeekV32ChatRenderer {

24 Ok(RenderedPrompt {

25 prompt: Prompt::Text(encoding::render_request(request)?),

26 + effective_template_kwargs: request_template_kwargs(request),

27 })

28 }

29 }

rust/src/chat/src/renderer/deepseek_v4/mod.rs

+2 -1

@@ -2,7 +2,7 @@ mod encoding;

3 use vllm_text::Prompt;

5 -use super::{ChatRenderer, RenderedPrompt};

5 +use super::{ChatRenderer, RenderedPrompt, request_template_kwargs};

6 use crate::Result;

7 use crate::request::ChatRequest;

@@ -22,6 +22,7 @@ impl ChatRenderer for DeepSeekV4ChatRenderer {

23 Ok(RenderedPrompt {

24 prompt: Prompt::Text(encoding::render_request(request)?),

25 + effective_template_kwargs: request_template_kwargs(request),

26 })

27 }

28 }

rust/src/chat/src/renderer/hf/mod.rs

+44 -7

@@ -14,7 +14,7 @@ use self::format::{

14 };

15 use self::template::{CompiledChatTemplate, TemplateContext};

16 use self::value::{TemplateValue, to_template_value};

17 -use super::{ChatRenderer, RenderedPrompt};

17 +use super::{ChatRenderer, RenderedPrompt, effective_template_kwargs};

18 use crate::error::Result;

19 use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};

20 use crate::{

@@ -169,8 +169,8 @@ impl HfChatRenderer {

169 "applying chat template"

170 );

171

172 - let mut merged_template_kwargs = self.default_template_kwargs.clone();

173 - merged_template_kwargs.extend(request.chat_options.template_kwargs.clone());

172 + let effective_template_kwargs =

173 + effective_template_kwargs(&self.default_template_kwargs, request);

174 let prompt = effective_template

175 .apply(TemplateContext {

176 messages: &messages,

@@ -178,9 +178,8 @@ impl HfChatRenderer {

178 continue_final_message: request.chat_options.continue_final_message(),

179 tools: tools.as_deref(),

180 documents: request.documents.as_deref(),

181 - template_kwargs: Some(&merged_template_kwargs),

181 + template_kwargs: Some(&effective_template_kwargs),

182 special_tokens: self.special_tokens.as_ref(),

183 - reasoning_effort: request.chat_options.reasoning_effort,

183 })

184 .map_err(|error| Error::ChatTemplate(error.to_report_string()))?;

185

@@ -191,6 +190,7 @@ impl HfChatRenderer {

190

191 Ok(RenderedPrompt {

192 prompt: Prompt::Text(prompt),

193 + effective_template_kwargs,

194 })

195 }

196 }

@@ -797,9 +797,46 @@ mod tests {

797 )

798 .unwrap();

799

800 - let rendered = renderer.render(&request).unwrap().prompt;

800 + let rendered = renderer.render(&request).unwrap();

801 +

802 + assert_eq!(rendered.prompt, Prompt::Text("max".to_string()));

803 + assert_eq!(

804 + rendered.effective_template_kwargs.get("reasoning_effort"),

805 + Some(&Value::String("max".to_string()))

806 + );

807 + assert_eq!(

808 + rendered.effective_template_kwargs.get("enable_thinking"),

809 + Some(&Value::Bool(true))

810 + );

811 + }

812

802 - assert_eq!(rendered, Prompt::Text("max".to_string()));

813 + #[test]

814 + fn chat_template_reasoning_effort_preserves_request_enable_thinking() {

815 + let mut request = sample_request(vec![ChatMessage::text(ChatRole::User, "hello")]);

816 + request.chat_options.reasoning_effort = Some(ReasoningEffort::None);

817 + request

818 + .chat_options

819 + .template_kwargs

820 + .insert("enable_thinking".to_string(), Value::Bool(true));

821 +

822 + let renderer = HfChatRenderer::new(

823 + Some("{{ reasoning_effort }}|{{ enable_thinking }}".to_string()),

824 + HashMap::new(),

825 + ChatTemplateContentFormatOption::Auto,

826 + )

827 + .unwrap();

828 +

829 + let rendered = renderer.render(&request).unwrap();

830 +

831 + assert_eq!(rendered.prompt, Prompt::Text("none|true".to_string()));

832 + assert_eq!(

833 + rendered.effective_template_kwargs.get("reasoning_effort"),

834 + Some(&Value::String("none".to_string()))

835 + );

836 + assert_eq!(

837 + rendered.effective_template_kwargs.get("enable_thinking"),

838 + Some(&Value::Bool(true))

839 + );

840 }

841

842 #[test]

rust/src/chat/src/renderer/hf/template.rs

-4

@@ -19,7 +19,6 @@ use super::format::{

19 };

20 use super::tojson::hf_tojson_filter;

21 use crate::renderer::hf::{TemplateMessage, TemplateTool};

22 -use crate::request::ReasoningEffort;

23 type Result<T> = std::result::Result<T, TemplateError>;

@@ -50,9 +49,6 @@ pub(super) struct TemplateContext<'a> {

49 pub(super) special_tokens: Option<&'a HfSpecialTokens>,

50 #[serde(flatten)]

51 pub(super) template_kwargs: Option<&'a HashMap<String, serde_json::Value>>,

53 - // By putting top-level `reasoning_effort` after `template_kwargs`, this overrides any

54 - // `reasoning_effort` value that might be present there.

55 - pub(super) reasoning_effort: Option<ReasoningEffort>,

52 }

54 /// Load chat template from a file (`.jinja` or `.json` containing Jinja).

rust/src/chat/src/renderer/mod.rs

+38 -2

@@ -1,9 +1,11 @@

1 +use std::collections::HashMap;

2 use std::sync::Arc;

4 +use serde_json::{Value, json};

5 use vllm_text::Prompt;

7 use crate::error::Result;

6 -use crate::request::ChatRequest;

8 +use crate::request::{ChatRequest, ReasoningEffort};

10 pub mod deepseek_v32;

11 pub mod deepseek_v4;

@@ -15,9 +17,13 @@ pub use deepseek_v32::DeepSeekV32ChatRenderer;

17 pub use selection::RendererSelection;

19 /// Rendered chat prompt submitted to the text backend.

18 -#[derive(Debug, Clone, PartialEq, Eq)]

20 +#[derive(Debug, Clone, PartialEq)]

21 pub struct RenderedPrompt {

22 + /// The rendered prompt, either as text or already tokenized.

23 pub prompt: Prompt,

24 + /// Effective chat-template kwargs visible to the renderer after applying

25 + /// server defaults, request overrides, and typed reasoning controls.

26 + pub effective_template_kwargs: HashMap<String, Value>,

27 }

29 /// Minimal chat-prompt renderer used by `vllm-chat`.

@@ -29,3 +35,33 @@ pub trait ChatRenderer: Send + Sync {

36 /// Shared trait-object form of [`ChatRenderer`].

37 pub type DynChatRenderer = Arc<dyn ChatRenderer>;

38 +

39 +/// Extract the effective chat-template kwargs visible to the renderer from the request,

40 +/// using the provided defaults as the base.

41 +pub(crate) fn effective_template_kwargs(

42 + default_template_kwargs: &HashMap<String, Value>,

43 + request: &ChatRequest,

44 +) -> HashMap<String, Value> {

45 + let mut kwargs = default_template_kwargs.clone();

46 + kwargs.extend(request.chat_options.template_kwargs.clone());

47 +

48 + if let Some(reasoning_effort) = request.chat_options.reasoning_effort {

49 + kwargs.insert(

50 + "reasoning_effort".to_string(),

51 + Value::String(reasoning_effort.as_str().to_string()),

52 + );

53 + if !request.chat_options.template_kwargs.contains_key("enable_thinking") {

54 + kwargs.insert(

55 + "enable_thinking".to_string(),

56 + json!(reasoning_effort != ReasoningEffort::None),

57 + );

58 + }

59 + }

60 +

61 + kwargs

62 +}

63 +

64 +/// Extract the effective chat-template kwargs visible to the renderer from the request.

65 +pub(crate) fn request_template_kwargs(request: &ChatRequest) -> HashMap<String, Value> {

66 + effective_template_kwargs(&HashMap::new(), request)

67 +}

rust/src/chat/tests/chat.rs

@@ -277,6 +277,7 @@ impl ChatRenderer for FakeChatBackend {

277

278 Ok(RenderedPrompt {

279 prompt: Prompt::Text(prompt),

280 + effective_template_kwargs: request.chat_options.template_kwargs.clone(),

281 })

282 }

283 }

rust/src/engine-core-client/src/protocol/mod.rs

+12 -2

@@ -372,6 +372,16 @@ impl EngineCoreSamplingParams {

372 }

373 }

374

375 +/// Extra kwargs consumed by engine-side reasoning parsers.

376 +///

377 +/// Original Python construction point:

378 +/// <https://github.com/vllm-project/vllm/blob/cec2ec11760f9f3beabd4c90451936078bf91533/vllm/entrypoints/openai/chat_completion/serving.py#L367-L369>

379 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]

380 +pub struct ReasoningParserKwargs {

381 + /// Effective kwargs visible to the chat template for this request.

382 + pub chat_template_kwargs: HashMap<String, serde_json::Value>,

383 +}

384 +

385 /// Engine-core add-request payload sent from frontend to engine.

386 ///

387 /// Original Python definition:

@@ -421,10 +431,10 @@ pub struct EngineCoreRequest {

431 pub external_req_id: Option<String>,

432 #[serde(default)]

433 pub reasoning_ended: Option<bool>,

424 - /// Opaque reasoning-parser kwargs forwarded from the frontend to the

434 + /// Reasoning-parser kwargs forwarded from the frontend to the

435 /// structured-output backend.

436 #[serde(default)]

427 - pub reasoning_parser_kwargs: Option<OpaqueValue>,

437 + pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,

438 /// If `true`, the request should be added to the scheduler's waiting queue

439 /// and immediately aborted, so connector-side cleanup runs via the

440 /// standard `request_finished` hook.

rust/src/llm/examples/external_engine_smoke.rs

+1 -1

@@ -56,7 +56,7 @@ fn build_request(request_id: String, max_tokens: u32) -> GenerateRequest {

56 trace_headers: None,

57 priority: 0,

58 data_parallel_rank: None,

59 - reasoning_ended: None,

59 + reasoning_parser_kwargs: None,

60 lora_request: None,

61 }

62 }

rust/src/llm/src/request.rs

+41 -11

@@ -4,7 +4,9 @@ use std::time::{SystemTime, UNIX_EPOCH};

4 use uuid::Uuid;

5 use vllm_engine_core_client::protocol::lora::LoraRequest;

6 use vllm_engine_core_client::protocol::multimodal::MmFeatures;

7 -use vllm_engine_core_client::protocol::{EngineCoreRequest, EngineCoreSamplingParams};

7 +use vllm_engine_core_client::protocol::{

8 + EngineCoreRequest, EngineCoreSamplingParams, ReasoningParserKwargs,

9 +};

11 use crate::error::{Error, Result};

@@ -27,14 +29,24 @@ pub struct GenerateRequest {

29 pub sampling_params: EngineCoreSamplingParams,

30 /// Optional multimodal features already prepared by `vllm-chat`.

31 pub mm_features: Option<MmFeatures>,

30 -

31 - // Fields below are currently likely unused by callers.

32 + /// Unix timestamp, in seconds, when this request arrived at the frontend.

33 + ///

34 + /// When omitted, the Rust frontend fills it immediately before sending the

35 + /// request to engine-core, matching Python's default arrival-time behavior.

36 pub arrival_time: Option<f64>,

37 + /// Optional salt used to partition prefix-cache entries for this request.

38 pub cache_salt: Option<String>,

39 + /// Optional tracing headers to forward to engine-core and downstream

40 + /// observability hooks.

41 pub trace_headers: Option<BTreeMap<String, String>>,

42 + /// Request scheduling priority. Lower values are scheduled earlier.

43 pub priority: i32,

44 + /// Optional data-parallel rank override for routing this request.

45 pub data_parallel_rank: Option<u32>,

37 - pub reasoning_ended: Option<bool>,

46 + /// Optional reasoning-parser kwargs forwarded to engine-side structured

47 + /// output logic.

48 + pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,

49 + /// Optional LoRA adapter request applied to this generation.

50 pub lora_request: Option<LoraRequest>,

51 }

@@ -61,7 +73,7 @@ impl GenerateRequest {

73 trace_headers,

74 priority,

75 data_parallel_rank,

64 - reasoning_ended,

76 + reasoning_parser_kwargs,

77 lora_request,

78 } = self;

@@ -72,7 +84,6 @@ impl GenerateRequest {

84 } else {

85 external_request_id.clone()

86 };

75 -

87 Ok(PreparedGenerateRequest {

88 engine_request: EngineCoreRequest {

89 request_id: engine_request_id,

@@ -92,8 +103,10 @@ impl GenerateRequest {

103 trace_headers,

104 resumable: false,

105 external_req_id: Some(external_request_id),

95 - reasoning_ended,

96 - reasoning_parser_kwargs: None,

106 + // Rust parser doesn't expose this information, leave it unset and let the

107 + // reasoning logic in engine-sided structured output manager handle it.

108 + reasoning_ended: None,

109 + reasoning_parser_kwargs,

110 abort_immediately: false,

111 },

112 })

@@ -121,7 +134,7 @@ fn current_unix_timestamp_secs() -> f64 {

134 mod tests {

135 use std::collections::BTreeMap;

136

124 - use vllm_engine_core_client::protocol::EngineCoreSamplingParams;

137 + use vllm_engine_core_client::protocol::{EngineCoreSamplingParams, ReasoningParserKwargs};

138

139 use super::GenerateRequest;

140 use crate::error::Error;

@@ -140,7 +153,15 @@ mod tests {

153 )])),

154 priority: 3,

155 data_parallel_rank: Some(2),

143 - reasoning_ended: Some(true),

156 + reasoning_parser_kwargs: Some(ReasoningParserKwargs {

157 + chat_template_kwargs: [(

158 + "chat_template_kwargs".to_string(),

159 + serde_json::json!({

160 + "enable_thinking": true,

161 + }),

162 + )]

163 + .into(),

164 + }),

165 lora_request: None,

166 }

167 }

@@ -166,7 +187,16 @@ mod tests {

187 "abc".to_string(),

188 )]))

189 );

169 - assert_eq!(request.reasoning_ended, Some(true));

190 + assert_eq!(request.reasoning_ended, None);

191 + assert_eq!(

192 + request

193 + .reasoning_parser_kwargs

194 + .as_ref()

195 + .and_then(|kwargs| kwargs.chat_template_kwargs.get("chat_template_kwargs")),

196 + Some(&serde_json::json!({

197 + "enable_thinking": true

198 + }))

199 + );

200 }

201

202 #[test]

rust/src/llm/tests/generate.rs

+1 -1

@@ -179,7 +179,7 @@ fn sample_generate_request(request_id: &str, max_tokens: u32) -> GenerateRequest

179 trace_headers: None,

180 priority: 0,

181 data_parallel_rank: None,

182 - reasoning_ended: None,

182 + reasoning_parser_kwargs: None,

183 lora_request: None,

184 }

185 }

rust/src/server/src/grpc/convert.rs

@@ -91,6 +91,7 @@ pub fn to_text_request(

91 cache_salt: kv.map(|k| &k.cache_salt).filter(|s| !s.is_empty()).cloned(),

92 add_special_tokens: true,

93 data_parallel_rank: None,

94 + reasoning_parser_kwargs: None,

95 lora_request: None,

96 })

97 }

rust/src/server/src/grpc/tests.rs

@@ -206,6 +206,7 @@ impl ChatRenderer for FakeTextBackend {

206 fn render(&self, _request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {

207 Ok(RenderedPrompt {

208 prompt: Prompt::Text(String::new()),

209 + effective_template_kwargs: Default::default(),

210 })

211 }

212 }

rust/src/server/src/routes/http_client_tests.rs

@@ -223,6 +223,7 @@ impl ChatRenderer for FakeChatBackend {

223 }

224 Ok(RenderedPrompt {

225 prompt: Prompt::Text(prompt),

226 + effective_template_kwargs: Default::default(),

227 })

228 }

229 }

rust/src/server/src/routes/inference/generate/convert.rs

@@ -68,6 +68,7 @@ pub(super) fn prepare_generate_request(

68 cache_salt: request.cache_salt,

69 add_special_tokens: false,

70 data_parallel_rank: ctx.data_parallel_rank,

71 + reasoning_parser_kwargs: None,

72 lora_request: lora_resolution.lora_request.clone(),

73 };

rust/src/server/src/routes/openai/completions/convert.rs

@@ -141,6 +141,7 @@ pub(super) fn prepare_completion_request(

141 cache_salt: request.cache_salt,

142 add_special_tokens: request.add_special_tokens,

143 data_parallel_rank: ctx.data_parallel_rank,

144 + reasoning_parser_kwargs: None,

145 lora_request: lora_resolution.lora_request.clone(),

146 };

147

rust/src/server/src/routes/tests.rs

@@ -581,6 +581,7 @@ impl ChatRenderer for FakeChatBackend {

581 }

582 Ok(vllm_chat::RenderedPrompt {

583 prompt: Prompt::Text(prompt),

584 + effective_template_kwargs: Default::default(),

585 })

586 }

587 }

rust/src/text/src/lower.rs

+1 -2

@@ -49,11 +49,10 @@ pub fn lower_text_request(

49 cache_salt: request.cache_salt.clone(),

50 priority: request.priority,

51 data_parallel_rank: request.data_parallel_rank,

52 + reasoning_parser_kwargs: request.reasoning_parser_kwargs.clone(),

53 lora_request: request.lora_request.clone(),

53 - // Fields below are currently placeholders.

54 arrival_time: None,

55 trace_headers: None,

56 - reasoning_ended: None,

56 };

58 Ok(PreparedTextRequest {

rust/src/text/src/request.rs

+6 -1

@@ -3,9 +3,9 @@ use std::collections::HashMap;

3 use enum_as_inner::EnumAsInner;

4 use serde::{Deserialize, Serialize};

5 use serde_json::Value;

6 -use vllm_engine_core_client::protocol::StructuredOutputsParams;

6 use vllm_engine_core_client::protocol::lora::LoraRequest;

7 use vllm_engine_core_client::protocol::multimodal::MmFeatures;

8 +use vllm_engine_core_client::protocol::{ReasoningParserKwargs, StructuredOutputsParams};

10 use crate::error::{Error, Result};

11 use crate::output::TextDecodeOptions;

@@ -174,6 +174,10 @@ pub struct TextRequest {

174 /// Override data parallel rank.

175 #[serde(default)]

176 pub data_parallel_rank: Option<u32>,

177 + /// Optional reasoning-parser kwargs forwarded to engine-side structured

178 + /// output logic.

179 + #[serde(default)]

180 + pub reasoning_parser_kwargs: Option<ReasoningParserKwargs>,

181 /// LoRA adapter selected for this request.

182 #[serde(default)]

183 pub lora_request: Option<LoraRequest>,

@@ -193,6 +197,7 @@ impl TextRequest {

197 cache_salt: None,

198 add_special_tokens: false,

199 data_parallel_rank: None,

200 + reasoning_parser_kwargs: None,

201 lora_request: None,

202 }

203 }