@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 142020c6c2

[Bugfix][Rust Frontend] Honor `add_generation_prompt` in DeepSeek V3.2/V4/V4.1 renderers (#56593)

Co-authored-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: YeonwooSung <neos960518@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Yeonwoo Sung committed Sep 15, 2026 at 19:03 UTC 142020c6c2c0d4fec4d36df28290e3d9b654a385
5 files changed +217 -24
rust/src/chat/src/renderer/deepseek.rs
+28 -19
@@ -224,21 +224,30 @@ pub(super) fn render_request_with_media_order(
224 }
225 }
226
227 - if (is_user_like_entry(message, current_render_index as usize, dialect)
228 - || (dialect == DsDialect::V4 && matches!(message, ChatMessage::System { .. })))
229 - && next_rendered_entry_is_assistant_or_end(
227 + if is_user_like_entry(message, current_render_index as usize, dialect)
228 + || (dialect == DsDialect::V4 && matches!(message, ChatMessage::System { .. }))
229 + {
230 + // Skip the opener only at the last rendered entry when
231 + // add_generation_prompt is off. Inspect the next rendered
232 + // entry, not the raw index: V4 merges trailing tool results.
233 + let should_write_transition = match next_rendered_entry(
234 request.messages.as_slice(),
235 message_index,
236 drop_historical_developers,
237 last_user_like_message_index,
234 - )
235 - {
236 - write_assistant_transition(
237 - &mut out,
238 - thinking_mode,
239 - drop_thinking,
240 - current_render_index >= last_user_render_index,
241 - );
238 + ) {
239 + Some(ChatMessage::Assistant { .. }) => true,
240 + None => request.chat_options.add_generation_prompt(),
241 + Some(_) => false,
242 + };
243 + if should_write_transition {
244 + write_assistant_transition(
245 + &mut out,
246 + thinking_mode,
247 + drop_thinking,
248 + current_render_index >= last_user_render_index,
249 + );
250 + }
251 }
252 }
253
@@ -420,14 +429,17 @@ fn is_historical_developer(
429 && last_user_like_message_index.is_some_and(|last_index| message_index < last_index)
430 }
431
423 -/// Return whether the next rendered entry is assistant, or there is no next
424 -/// entry.
425 -fn next_rendered_entry_is_assistant_or_end(
432 +/// Return the next rendered message after `message_index`.
433 +///
434 +/// Skips merged user-content tails and dropped historical developers so the
435 +/// caller can tell a following assistant turn apart from the end of the
436 +/// conversation.
437 +fn next_rendered_entry(
438 messages: &[ChatMessage],
439 message_index: usize,
440 drop_historical_developers: bool,
441 last_user_like_message_index: Option<usize>,
430 -) -> bool {
442 +) -> Option<&ChatMessage> {
443 let mut next_index = message_index + 1;
444 while next_index < messages.len()
445 && (is_following_user_content(messages, next_index)
@@ -441,10 +453,7 @@ fn next_rendered_entry_is_assistant_or_end(
453 next_index += 1;
454 }
455
444 - messages
445 - .get(next_index)
446 - .map(|message| matches!(message, ChatMessage::Assistant { .. }))
447 - .unwrap_or(true)
456 + messages.get(next_index)
457 }
458
459 /// Render the tool preamble shown to the model for one DeepSeek dialect.
rust/src/chat/src/renderer/deepseek_v32/encoding.rs
+14 -4
@@ -61,6 +61,7 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
61 find_last_user_render_index(request.messages.as_slice(), render_offset);
62 let last_user_actual_index = find_last_user_actual_index(request.messages.as_slice());
63 let continue_final_message = request.chat_options.continue_final_message();
64 + let add_generation_prompt = request.chat_options.add_generation_prompt();
65 let mut prompt = String::from(BOS_TOKEN);
66
67 if !initial_tools.is_empty() {
@@ -79,6 +80,7 @@ pub(super) fn render_request(request: &ChatRequest) -> Result<String> {
80 thinking_mode,
81 drop_thinking,
82 continue_final_message,
83 + add_generation_prompt,
84 )?;
85 }
86
@@ -110,11 +112,13 @@ fn render_message(
112 thinking_mode: ThinkingMode,
113 drop_thinking: bool,
114 continue_final_message: bool,
115 + add_generation_prompt: bool,
116 ) -> Result<()> {
117 let render_index = message_index as isize + render_offset;
118 let opens_thinking = render_index == last_user_render_index;
119 let after_last_user_turn = render_index > last_user_render_index;
120 let after_or_at_last_user_turn = render_index >= last_user_render_index;
121 + let add_generation_prompt = message_index + 1 != messages.len() || add_generation_prompt;
122
123 match message {
124 ChatMessage::System { content } => render_system_message(out, Some(content), &[]),
@@ -123,11 +127,13 @@ fn render_message(
127 content,
128 tools.as_deref().unwrap_or(&[]),
129 thinking_mode == ThinkingMode::Thinking && opens_thinking,
130 + add_generation_prompt,
131 ),
132 ChatMessage::User { content } => render_user_message(
133 out,
134 content,
135 thinking_mode == ThinkingMode::Thinking && opens_thinking,
136 + add_generation_prompt,
137 ),
138 ChatMessage::Assistant { content } => render_assistant_message(
139 out,
@@ -195,6 +201,7 @@ fn render_developer_message(
201 content: &ChatContent,
202 tools: &[ChatTool],
203 opens_thinking: bool,
204 + add_generation_prompt: bool,
205 ) -> Result<()> {
206 if content.is_empty() {
207 return Err(Error::ChatTemplate(
@@ -209,7 +216,7 @@ fn render_developer_message(
216 }
217 out.push_str("\n\n# The user's message is: ");
218 write_chat_content(out, content)?;
212 - write_user_like_suffix(out, opens_thinking);
219 + write_user_like_suffix(out, opens_thinking, add_generation_prompt);
220 Ok(())
221 }
222
@@ -219,17 +226,20 @@ fn render_user_message(
226 out: &mut String,
227 content: &ChatContent,
228 opens_thinking: bool,
229 + add_generation_prompt: bool,
230 ) -> Result<()> {
231 out.push_str("<｜User｜>");
232 write_chat_content(out, content)?;
225 - write_user_like_suffix(out, opens_thinking);
233 + write_user_like_suffix(out, opens_thinking, add_generation_prompt);
234 Ok(())
235 }
236
237 /// Shared trailing wrapper used by both real user turns and native developer
238 /// turns after their content has already been written.
231 -// TODO: respect `add_generation_prompt` option
232 -fn write_user_like_suffix(out: &mut String, opens_thinking: bool) {
239 +fn write_user_like_suffix(out: &mut String, opens_thinking: bool, add_generation_prompt: bool) {
240 + if !add_generation_prompt {
241 + return;
242 + }
243 out.push_str("<｜Assistant｜>");
244 if opens_thinking {
245 out.push_str(THINKING_START_TOKEN);
rust/src/chat/src/renderer/deepseek_v32/tests.rs
+38
@@ -281,6 +281,44 @@ fn assistant_after_last_user_requires_reasoning_or_tool_calls() {
281 .assert_eq(&error.to_report_string());
282 }
283
284 +#[test]
285 +fn last_user_turn_appends_generation_prompt_by_default() {
286 + let request = thinking_request(vec![ChatMessage::user("Hello")]);
287 +
288 + let rendered = render_request(&request);
289 +
290 + assert!(rendered.ends_with("<｜User｜>Hello<｜Assistant｜><think>"));
291 +}
292 +
293 +#[test]
294 +fn last_user_turn_omits_generation_prompt_when_disabled() {
295 + let mut request = thinking_request(vec![ChatMessage::user("Hello")]);
296 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
297 +
298 + let rendered = render_request(&request);
299 +
300 + assert!(rendered.ends_with("<｜User｜>Hello"));
301 + assert!(!rendered.contains("<｜Assistant｜>"));
302 + assert!(!rendered.contains("<think>"));
303 +}
304 +
305 +#[test]
306 +fn user_then_assistant_still_writes_suffix_without_generation_prompt() {
307 + let mut request = ChatRequest {
308 + messages: vec![
309 + ChatMessage::user("Hello"),
310 + ChatMessage::assistant_text("Hi there."),
311 + ],
312 + ..ChatRequest::for_test()
313 + };
314 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
315 +
316 + let rendered = render_request(&request);
317 +
318 + expect!["<｜begin▁of▁sentence｜><｜User｜>Hello<｜Assistant｜></think>Hi there.<｜end▁of▁sentence｜>"]
319 + .assert_eq(&rendered);
320 +}
321 +
322 #[test]
323 fn continue_final_assistant_omits_final_eos() {
324 let mut request = ChatRequest {
rust/src/chat/src/renderer/deepseek_v4/tests.rs
+91
@@ -633,3 +633,94 @@ fn continue_final_assistant_omits_final_eos() {
633 expect!["<｜begin▁of▁sentence｜><｜User｜>write<｜Assistant｜></think>partial answer"]
634 .assert_eq(&rendered);
635 }
636 +
637 +#[test]
638 +fn last_user_turn_appends_generation_prompt_by_default() {
639 + let request = thinking_request(vec![ChatMessage::user("Hello")]);
640 +
641 + let rendered = render_request(&request);
642 +
643 + assert!(rendered.ends_with("<｜User｜>Hello<｜Assistant｜><think>"));
644 +}
645 +
646 +#[test]
647 +fn last_user_turn_omits_generation_prompt_when_disabled() {
648 + let mut request = thinking_request(vec![ChatMessage::user("Hello")]);
649 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
650 +
651 + let rendered = render_request(&request);
652 +
653 + assert!(rendered.ends_with("<｜User｜>Hello"));
654 + assert!(!rendered.contains("<｜Assistant｜>"));
655 + assert!(!rendered.contains("<think>"));
656 +}
657 +
658 +#[test]
659 +fn user_then_assistant_still_writes_suffix_without_generation_prompt() {
660 + let mut request = ChatRequest {
661 + messages: vec![
662 + ChatMessage::user("Hello"),
663 + ChatMessage::assistant_text("Hi there."),
664 + ],
665 + ..ChatRequest::for_test()
666 + };
667 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
668 + request
669 + .chat_options
670 + .template_kwargs
671 + .insert("thinking".to_string(), Value::Bool(false));
672 +
673 + let rendered = render_request(&request);
674 +
675 + expect![
676 + "<｜begin▁of▁sentence｜><｜User｜>Hello<｜Assistant｜></think>Hi there.<｜end▁of▁sentence｜>"
677 + ]
678 + .assert_eq(&rendered);
679 +}
680 +
681 +#[test]
682 +fn trailing_system_omits_generation_prompt_when_disabled() {
683 + let mut request = ChatRequest {
684 + messages: vec![
685 + ChatMessage::system("rules"),
686 + ChatMessage::user("question"),
687 + ChatMessage::system("final rules"),
688 + ],
689 + ..ChatRequest::for_test()
690 + };
691 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
692 + request
693 + .chat_options
694 + .template_kwargs
695 + .insert("enable_thinking".to_string(), Value::Bool(false));
696 + request.chat_options.reasoning_effort = Some(ReasoningEffort::Low);
697 +
698 + let rendered = render_request(&request);
699 +
700 + expect!["<｜begin▁of▁sentence｜>rules<｜User｜>questionfinal rules"].assert_eq(&rendered);
701 +}
702 +
703 +#[test]
704 +fn last_merged_tool_result_omits_generation_prompt_when_disabled() {
705 + let mut request = thinking_request(vec![
706 + ChatMessage::user("question"),
707 + ChatMessage::tool_response("result", "call-1"),
708 + ]);
709 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
710 +
711 + let rendered = render_request(&request);
712 +
713 + expect!["<｜begin▁of▁sentence｜><｜User｜>question\n\n<tool_result>result</tool_result>"]
714 + .assert_eq(&rendered);
715 + assert!(!rendered.contains("<｜Assistant｜>"));
716 +}
717 +
718 +#[test]
719 +fn last_developer_omits_generation_prompt_when_disabled() {
720 + let mut request = thinking_request(vec![ChatMessage::developer("latest instruction", None)]);
721 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
722 +
723 + let rendered = render_request(&request);
724 +
725 + expect!["<｜begin▁of▁sentence｜><｜User｜>latest instruction"].assert_eq(&rendered);
726 +}
rust/src/chat/src/renderer/deepseek_v41/tests.rs
+46 -1
@@ -10,7 +10,9 @@ use super::DeepSeekV41ChatRenderer;
10 use crate::ChatRenderer;
11 use crate::event::{AssistantContentBlock, AssistantToolCall};
12 use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
13 -use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ReasoningEffort};
13 +use crate::request::{
14 + ChatContent, ChatContentPart, ChatMessage, ChatRequest, GenerationPromptMode, ReasoningEffort,
15 +};
16
17 fn render(request: &ChatRequest) -> String {
18 DeepSeekV41ChatRenderer::new()
@@ -281,6 +283,49 @@ fn mid_system_advances_last_user_boundary_and_drops_old_reasoning() {
283 .assert_eq(&render(&request));
284 }
285
286 +#[test]
287 +fn last_user_turn_omits_generation_prompt_when_disabled() {
288 + let mut request = request();
289 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
290 +
291 + expect!["<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 50 (range 1-100, the higher the value, the more thorough the reasoning)\n\n<｜User｜>question"]
292 + .assert_eq(&render(&request));
293 +}
294 +
295 +#[test]
296 +fn last_mid_system_omits_generation_prompt_when_disabled() {
297 + let mut request = ChatRequest {
298 + messages: vec![
299 + ChatMessage::system("sys"),
300 + ChatMessage::user("q1"),
301 + ChatMessage::assistant_blocks(vec![
302 + AssistantContentBlock::Reasoning { text: "r1".into() },
303 + AssistantContentBlock::Text { text: "a1".into() },
304 + ]),
305 + ChatMessage::developer("old policy", None),
306 + ChatMessage::system("mid sys"),
307 + ],
308 + ..ChatRequest::for_test()
309 + };
310 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
311 + request
312 + .chat_options
313 + .template_kwargs
314 + .insert("reasoning_effort".into(), json!(88));
315 +
316 + expect!["<｜begin▁of▁sentence｜><｜System｜>Reasoning Effort: 88 (range 1-100, the higher the value, the more thorough the reasoning)\n\nsys<｜User｜>q1<｜Assistant｜></think>a1<｜end▁of▁sentence｜><｜System｜>mid sys"]
317 + .assert_eq(&render(&request));
318 +}
319 +
320 +#[test]
321 +fn chat_mode_omits_generation_prompt_when_disabled() {
322 + let mut request = request();
323 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
324 + request.chat_options.template_kwargs.insert("thinking".into(), json!(false));
325 +
326 + expect!["<｜begin▁of▁sentence｜><｜User｜>question"].assert_eq(&render(&request));
327 +}
328 +
329 #[test]
330 fn joins_text_parts_with_reference_separator_and_preserves_literal_tags() {
331 let mut request = ChatRequest {