@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 7b1ea3f524

[Bugfix][Rust Frontend][Multimodal] Align DeepSeek V4.1 and Kimi K3 media with rendered placeholders (#56366)

Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Codex <noreply@openai.com> Signed-off-by: reidliu41 <reid201711@gmail.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Reid committed Sep 14, 2026 at 18:37 UTC 7b1ea3f524445be0545ed962cb06ce21a93fcc33
18 files changed +415 -85
rust/src/chat/src/lib.rs
+7 -3
@@ -34,8 +34,8 @@ pub use parser::{ParserSelection, validate_parser_overrides};
34 pub use renderer::hf::ChatTemplateContentFormatOption;
35 pub use renderer::{
36 ChatRenderer, DeepSeekV4ChatRenderer, DeepSeekV32ChatRenderer, DeepSeekV41ChatRenderer,
37 - DynChatRenderer, HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, RenderedPrompt,
38 - RendererSelection,
37 + DynChatRenderer, HarmonyChatRenderer, InklingChatRenderer, KimiK3ChatRenderer, MediaPartSource,
38 + RenderedPrompt, RendererSelection,
39 };
40 pub use request::{
41 ChatContent, ChatContentPart, ChatMessage, ChatOptions, ChatRequest, ChatRole, ChatTool,
@@ -112,6 +112,10 @@ impl ChatRequestProcessor {
112 request: &ChatRequest,
113 rendered: RenderedPrompt,
114 ) -> Result<(Prompt, Option<MmFeatures>)> {
115 + let media_is_empty = match &rendered.media_order {
116 + Some(media_order) => media_order.is_empty(),
117 + None => !request.has_multimodal(),
118 + };
119 match self.model_dtype {
120 Some(model_dtype) => {
121 multimodal::finalize_rendered_prompt(
@@ -122,7 +126,7 @@ impl ChatRequestProcessor {
126 )
127 .await
128 }
125 - None if !request.has_multimodal() => Ok((rendered.prompt, None)),
129 + None if media_is_empty => Ok((rendered.prompt, None)),
130 None => Err(Error::UnsupportedMultimodalRenderer),
131 }
132 }
rust/src/chat/src/multimodal.rs
+152 -53
@@ -16,7 +16,7 @@ use std::fs;
16 use std::path::Path;
17 use std::sync::{Arc, LazyLock};
18
19 -use itertools::izip;
19 +use itertools::{Either, izip};
20 use llm_multimodal::{
21 AsyncMultiModalTracker, AudioClip, AudioPreProcessor, EncoderFieldLayouts, FieldLayout,
22 ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata,
@@ -33,7 +33,7 @@ use vllm_text::Prompt;
33 use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
34
35 use crate::error::{Error, Result, bail_multimodal, multimodal};
36 -use crate::renderer::RenderedPrompt;
36 +use crate::renderer::{MediaPartSource, RenderedPrompt};
37 use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
38
39 mod audio;
@@ -549,7 +549,8 @@ pub(crate) async fn finalize_rendered_prompt(
549 info: Option<&MultimodalModelInfo>,
550 model_dtype: ModelDtype,
551 ) -> Result<(Prompt, Option<MmFeatures>)> {
552 - if !request.has_multimodal() {
552 + let media_parts = extract_media_parts(request, rendered.media_order.as_deref())?;
553 + if media_parts.is_empty() {
554 return Ok((rendered.prompt, None));
555 }
556 let info = info.ok_or(Error::UnsupportedMultimodalRenderer)?;
@@ -561,63 +562,92 @@ pub(crate) async fn finalize_rendered_prompt(
562 .map_err(|error| multimodal!("{error}"))?,
563 Prompt::TokenIds(token_ids) => token_ids,
564 };
564 - let media_parts = extract_media_parts(request)?;
565 let prepared = info.prepare_multimodal(media_parts, &mut prompt_token_ids, model_dtype).await?;
566
567 Ok((Prompt::TokenIds(prompt_token_ids), Some(prepared)))
568 }
569
570 -/// Extract media parts from chat messages in message/content order.
571 -///
572 -/// Assistant history is skipped because generated assistant blocks are already
573 -/// represented as text for prompt rendering in this crate.
574 -fn extract_media_parts(request: &ChatRequest) -> Result<Vec<MediaContentPart>> {
575 - let mut all_parts = Vec::new();
576 - for message in &request.messages {
577 - let content = match message {
578 - ChatMessage::System { content }
579 - | ChatMessage::Developer { content, .. }
580 - | ChatMessage::User { content }
581 - | ChatMessage::ToolResponse { content, .. } => content,
582 - ChatMessage::Assistant { .. } => continue,
583 - };
584 - let ChatContent::Parts(parts) = content else {
585 - continue;
586 - };
587 - for part in parts {
588 - match part {
589 - ChatContentPart::Text { .. } => {}
590 - ChatContentPart::ImageUrl {
591 - image_url,
592 - detail,
593 - uuid,
594 - } => all_parts.push(MediaContentPart::ImageUrl {
595 - url: image_url.clone(),
596 - detail: *detail,
597 - uuid: uuid.clone(),
598 - }),
599 - ChatContentPart::VideoUrl { video_url, uuid } => {
600 - all_parts.push(MediaContentPart::VideoUrl {
601 - url: video_url.clone(),
602 - uuid: uuid.clone(),
603 - })
604 - }
605 - ChatContentPart::InputAudio { data, format, uuid } => {
606 - all_parts.push(MediaContentPart::AudioUrl {
607 - url: input_audio_data_url(data, format.as_deref())?,
608 - uuid: uuid.clone(),
609 - })
610 - }
611 - ChatContentPart::AudioUrl { audio_url, uuid } => {
612 - all_parts.push(MediaContentPart::AudioUrl {
613 - url: audio_url.clone(),
614 - uuid: uuid.clone(),
615 - })
570 +/// Resolve media parts in the placeholder order reported by the renderer.
571 +fn extract_media_parts(
572 + request: &ChatRequest,
573 + media_order: Option<&[MediaPartSource]>,
574 +) -> Result<Vec<MediaContentPart>> {
575 + let parts = match media_order {
576 + // Resolve exactly the sources selected by the renderer, in placeholder order.
577 + Some(media_order) => Either::Left(media_order.iter().map(|source| -> Result<_> {
578 + let message = request.messages.get(source.message_index).ok_or_else(|| {
579 + multimodal!(
580 + "renderer reported missing chat message {} for multimodal content",
581 + source.message_index
582 + )
583 + })?;
584 + let content = match message {
585 + ChatMessage::System { content }
586 + | ChatMessage::Developer { content, .. }
587 + | ChatMessage::User { content }
588 + | ChatMessage::ToolResponse { content, .. } => content,
589 + ChatMessage::Assistant { .. } => {
590 + bail_multimodal!("renderer reported multimodal assistant content")
591 }
592 + };
593 + let ChatContent::Parts(parts) = content else {
594 + bail_multimodal!("renderer reported multimodal content in a text-only chat message")
595 + };
596 + let part = parts.get(source.content_part_index).ok_or_else(|| {
597 + multimodal!(
598 + "renderer reported missing content part {} in chat message {}",
599 + source.content_part_index,
600 + source.message_index
601 + )
602 + })?;
603 + Ok(part)
604 + })),
605 + // Fall back to all media in request message and content-part order (e.g. Jinja).
606 + None => Either::Right(request.messages.iter().flat_map(|message| {
607 + let content = match message {
608 + ChatMessage::System { content }
609 + | ChatMessage::Developer { content, .. }
610 + | ChatMessage::User { content }
611 + | ChatMessage::ToolResponse { content, .. } => Some(content),
612 + ChatMessage::Assistant { .. } => None,
613 + };
614 + match content {
615 + Some(ChatContent::Parts(parts)) => parts.as_slice(),
616 + _ => &[],
617 }
618 - }
619 - }
620 - Ok(all_parts)
618 + .iter()
619 + .filter(|part| part.is_multimodal())
620 + .map(Ok)
621 + })),
622 + };
623 + parts
624 + .map(|part| match part? {
625 + ChatContentPart::Text { .. } => {
626 + bail_multimodal!("renderer reported a text part as multimodal content")
627 + }
628 + ChatContentPart::ImageUrl {
629 + image_url,
630 + detail,
631 + uuid,
632 + } => Ok(MediaContentPart::ImageUrl {
633 + url: image_url.clone(),
634 + detail: *detail,
635 + uuid: uuid.clone(),
636 + }),
637 + ChatContentPart::VideoUrl { video_url, uuid } => Ok(MediaContentPart::VideoUrl {
638 + url: video_url.clone(),
639 + uuid: uuid.clone(),
640 + }),
641 + ChatContentPart::InputAudio { data, format, uuid } => Ok(MediaContentPart::AudioUrl {
642 + url: input_audio_data_url(data, format.as_deref())?,
643 + uuid: uuid.clone(),
644 + }),
645 + ChatContentPart::AudioUrl { audio_url, uuid } => Ok(MediaContentPart::AudioUrl {
646 + url: audio_url.clone(),
647 + uuid: uuid.clone(),
648 + }),
649 + })
650 + .collect()
651 }
652
653 /// Wrap OpenAI base64 audio in a data URL consumed by `MediaConnector`.
@@ -1043,6 +1073,75 @@ mod tests {
1073 assert!(input_audio_data_url("AAAA", Some("flac")).is_err());
1074 }
1075
1076 + #[test]
1077 + fn extract_media_parts_uses_request_order_or_explicit_selection() {
1078 + let request = ChatRequest {
1079 + messages: vec![
1080 + ChatMessage::user(vec![
1081 + ChatContentPart::text("before image"),
1082 + ChatContentPart::ImageUrl {
1083 + image_url: "data:image/png;base64,image-b".to_string(),
1084 + detail: None,
1085 + uuid: Some("image-b".to_string()),
1086 + },
1087 + ]),
1088 + ChatMessage::user(vec![ChatContentPart::ImageUrl {
1089 + image_url: "data:image/png;base64,image-a".to_string(),
1090 + detail: None,
1091 + uuid: Some("image-a".to_string()),
1092 + }]),
1093 + ChatMessage::assistant_text("answer"),
1094 + ChatMessage::user("follow-up"),
1095 + ],
1096 + ..ChatRequest::for_test()
1097 + };
1098 + let media_order = [
1099 + MediaPartSource {
1100 + message_index: 1,
1101 + content_part_index: 0,
1102 + },
1103 + MediaPartSource {
1104 + message_index: 0,
1105 + content_part_index: 1,
1106 + },
1107 + ];
1108 +
1109 + let selections = [None, Some(&media_order[..]), Some(&[][..])];
1110 + let uuids = selections.map(|order| {
1111 + extract_media_parts(&request, order)
1112 + .unwrap()
1113 + .into_iter()
1114 + .map(|part| match part {
1115 + MediaContentPart::ImageUrl { uuid, .. } => uuid,
1116 + _ => panic!("expected image URL"),
1117 + })
1118 + .collect::<Vec<_>>()
1119 + });
1120 +
1121 + expect_test::expect![[r#"
1122 + [
1123 + [
1124 + Some(
1125 + "image-b",
1126 + ),
1127 + Some(
1128 + "image-a",
1129 + ),
1130 + ],
1131 + [
1132 + Some(
1133 + "image-a",
1134 + ),
1135 + Some(
1136 + "image-b",
1137 + ),
1138 + ],
1139 + [],
1140 + ]
1141 + "#]]
1142 + .assert_debug_eq(&uuids);
1143 + }
1144 +
1145 fn image_url_part() -> MediaContentPart {
1146 MediaContentPart::ImageUrl {
1147 url: "https://example.com/image.png".to_string(),
rust/src/chat/src/renderer/deepseek.rs
+53 -12
@@ -16,6 +16,7 @@ use serde_json_fmt::JsonFormat;
16
17 use llm_multimodal::DEEPSEEK_V41_IMAGE_PLACEHOLDER;
18
19 +use super::MediaPartSource;
20 use crate::error::{Error, Result};
21 use crate::request::{
22 ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ReasoningEffort,
@@ -87,6 +88,14 @@ struct RenderedToolSchema<'a> {
88
89 /// Render one chat request into the final prompt string.
90 pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Result<String> {
91 + render_request_with_media_order(request, dialect).map(|(prompt, _)| prompt)
92 +}
93 +
94 +/// Render one chat request and record media in placeholder order.
95 +pub(super) fn render_request_with_media_order(
96 + request: &ChatRequest,
97 + dialect: DsDialect,
98 +) -> Result<(String, Vec<MediaPartSource>)> {
99 let (thinking_mode, reasoning_effort_prompt) = match dialect {
100 DsDialect::V4 => {
101 resolve_thinking_options(request).map(|(mode, prompt)| (mode, Cow::Borrowed(prompt)))?
@@ -112,6 +121,7 @@ pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Resul
121 dialect,
122 );
123 let mut out = String::from(BOS_TOKEN);
124 + let mut media_order = Vec::new();
125 if dialect == DsDialect::V41
126 && (thinking_mode == ThinkingMode::Thinking
127 || synthetic_tool_system
@@ -126,7 +136,7 @@ pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Resul
136 let mut request_tools_attached = false;
137 let mut render_index = 0isize;
138 if synthetic_tool_system {
129 - render_system_message(&mut out, None, request_tools, dialect)?;
139 + render_system_message(&mut out, None, request_tools, dialect, &mut media_order)?;
140 request_tools_attached = true;
141 render_index += 1;
142 }
@@ -157,14 +167,22 @@ pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Resul
167 } else {
168 &[]
169 };
160 - render_system_message(&mut out, Some(content), tools, dialect)?;
170 + render_system_message(
171 + &mut out,
172 + Some((message_index, content)),
173 + tools,
174 + dialect,
175 + &mut media_order,
176 + )?;
177 }
178 ChatMessage::Developer { content, tools } => {
179 render_developer_message(
180 &mut out,
181 + message_index,
182 content,
183 tools.as_deref().unwrap_or(&[]),
184 dialect,
185 + &mut media_order,
186 )?;
187 }
188 ChatMessage::User { .. } | ChatMessage::ToolResponse { .. } => {
@@ -174,6 +192,7 @@ pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Resul
192 message_index,
193 &last_tool_call_order,
194 dialect,
195 + &mut media_order,
196 )?;
197 }
198 ChatMessage::Assistant { content } => {
@@ -223,7 +242,7 @@ pub(super) fn render_request(request: &ChatRequest, dialect: DsDialect) -> Resul
242 }
243 }
244
226 - Ok(out)
245 + Ok((out, media_order))
246 }
247
248 /// Resolve DeepSeek V4's thinking controls. Unlike the Python tokenizer
@@ -488,12 +507,13 @@ fn render_tool_schema(out: &mut String, tool: &ChatTool) -> Result<()> {
507 /// Render a system turn, optionally followed by the V4 tool preamble.
508 fn render_system_message(
509 out: &mut String,
491 - content: Option<&ChatContent>,
510 + content: Option<(usize, &ChatContent)>,
511 tools: &[ChatTool],
512 dialect: DsDialect,
513 + media_order: &mut Vec<MediaPartSource>,
514 ) -> Result<()> {
495 - if let Some(content) = content {
496 - write_chat_content(out, content, dialect)?;
515 + if let Some((message_index, content)) = content {
516 + write_chat_content(out, content, dialect, message_index, media_order)?;
517 }
518 if !tools.is_empty() {
519 out.push_str("\n\n");
@@ -505,9 +525,11 @@ fn render_system_message(
525 /// Developer messages are rendered as user-like turns with optional tools.
526 fn render_developer_message(
527 out: &mut String,
528 + message_index: usize,
529 content: &ChatContent,
530 tools: &[ChatTool],
531 dialect: DsDialect,
532 + media_order: &mut Vec<MediaPartSource>,
533 ) -> Result<()> {
534 if content.is_empty() {
535 return Err(Error::ChatTemplate(
@@ -516,7 +538,7 @@ fn render_developer_message(
538 }
539
540 out.push_str(USER_SP_TOKEN);
519 - write_chat_content(out, content, dialect)?;
541 + write_chat_content(out, content, dialect, message_index, media_order)?;
542 if !tools.is_empty() {
543 out.push_str("\n\n");
544 render_tools(out, tools, dialect)?;
@@ -531,6 +553,7 @@ fn render_user_content_block(
553 message_index: usize,
554 tool_call_order: &HashMap<String, usize>,
555 dialect: DsDialect,
556 + media_order: &mut Vec<MediaPartSource>,
557 ) -> Result<()> {
558 let (block_start, block_end) = user_content_block_bounds(messages, message_index);
559 let mut sorted_tool_indices =
@@ -542,7 +565,9 @@ fn render_user_content_block(
565 out.push_str("\n\n");
566 }
567 match &messages[message_index] {
545 - ChatMessage::User { content } => write_chat_content(out, content, dialect)?,
568 + ChatMessage::User { content } => {
569 + write_chat_content(out, content, dialect, message_index, media_order)?;
570 + }
571 ChatMessage::ToolResponse { .. } => {
572 let sorted_index = sorted_tool_indices
573 .next()
@@ -550,7 +575,7 @@ fn render_user_content_block(
575 let ChatMessage::ToolResponse { content, .. } = &messages[sorted_index] else {
576 unreachable!("sorted tool response index should reference a tool message");
577 };
553 - write_tool_result(out, content, dialect)?;
578 + write_tool_result(out, content, dialect, sorted_index, media_order)?;
579 }
580 _ => unreachable!("user content block should only contain user content messages"),
581 }
@@ -597,9 +622,15 @@ fn sorted_tool_response_indices(
622 }
623
624 /// Render one tool response payload inside a V4 `<tool_result>` block.
600 -fn write_tool_result(out: &mut String, content: &ChatContent, dialect: DsDialect) -> Result<()> {
625 +fn write_tool_result(
626 + out: &mut String,
627 + content: &ChatContent,
628 + dialect: DsDialect,
629 + message_index: usize,
630 + media_order: &mut Vec<MediaPartSource>,
631 +) -> Result<()> {
632 out.push_str("<tool_result>");
602 - write_chat_content(out, content, dialect)?;
633 + write_chat_content(out, content, dialect, message_index, media_order)?;
634 out.push_str("</tool_result>");
635 Ok(())
636 }
@@ -727,7 +758,13 @@ fn encode_arguments_to_dsml(
758 /// The V4.1 dialect inlines the image placeholder at each image part's
759 /// position unconditionally (matching the Python encoding's
760 /// `IMAGE_PLACEHOLDER`); other dialects reject multimodal parts.
730 -fn write_chat_content(out: &mut String, content: &ChatContent, dialect: DsDialect) -> Result<()> {
761 +fn write_chat_content(
762 + out: &mut String,
763 + content: &ChatContent,
764 + dialect: DsDialect,
765 + message_index: usize,
766 + media_order: &mut Vec<MediaPartSource>,
767 +) -> Result<()> {
768 match content {
769 ChatContent::Text(text) => out.push_str(text),
770 ChatContent::Parts(parts) => {
@@ -738,6 +775,10 @@ fn write_chat_content(out: &mut String, content: &ChatContent, dialect: DsDialec
775 match part {
776 ChatContentPart::ImageUrl { .. } if dialect == DsDialect::V41 => {
777 out.push_str(DEEPSEEK_V41_IMAGE_PLACEHOLDER);
778 + media_order.push(MediaPartSource {
779 + message_index,
780 + content_part_index: index,
781 + });
782 }
783 _ => out.push_str(part.as_text()?),
784 }
rust/src/chat/src/renderer/deepseek_v32/mod.rs
+1
@@ -26,6 +26,7 @@ impl ChatRenderer for DeepSeekV32ChatRenderer {
26
27 Ok(RenderedPrompt {
28 prompt: Prompt::Text(encoding::render_request(request)?),
29 + media_order: None,
30 effective_template_kwargs: request_template_kwargs(request),
31 })
32 }
rust/src/chat/src/renderer/deepseek_v4/mod.rs
+1
@@ -24,6 +24,7 @@ impl ChatRenderer for DeepSeekV4ChatRenderer {
24
25 Ok(RenderedPrompt {
26 prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V4)?),
27 + media_order: None,
28 effective_template_kwargs: request_template_kwargs(request),
29 })
30 }
rust/src/chat/src/renderer/deepseek_v41/mod.rs
+4 -1
@@ -21,9 +21,12 @@ impl DeepSeekV41ChatRenderer {
21 impl ChatRenderer for DeepSeekV41ChatRenderer {
22 fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
23 request.validate()?;
24 + let (prompt, media_order) =
25 + deepseek::render_request_with_media_order(request, DsDialect::V41)?;
26
27 Ok(RenderedPrompt {
26 - prompt: Prompt::Text(deepseek::render_request(request, DsDialect::V41)?),
28 + prompt: Prompt::Text(prompt),
29 + media_order: Some(media_order),
30 effective_template_kwargs: request_template_kwargs(request),
31 })
32 }
rust/src/chat/src/renderer/deepseek_v41/tests.rs
+77 -1
@@ -8,7 +8,7 @@ use serde_json::{Value, json};
8
9 use super::DeepSeekV41ChatRenderer;
10 use crate::ChatRenderer;
11 -use crate::event::AssistantContentBlock;
11 +use crate::event::{AssistantContentBlock, AssistantToolCall};
12 use crate::renderer::test_utils::{FixtureRequestOptions, fixture_chat_request};
13 use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest, ReasoningEffort};
14
@@ -83,6 +83,82 @@ fn inlines_image_placeholder_at_image_part_positions() {
83 );
84 }
85
86 +#[test]
87 +fn media_order_follows_reordered_tool_results() {
88 + let renderer = DeepSeekV41ChatRenderer::new();
89 + let request = ChatRequest {
90 + messages: vec![
91 + ChatMessage::assistant_blocks(vec![
92 + AssistantContentBlock::ToolCall(AssistantToolCall {
93 + id: "call-a".to_string(),
94 + name: "tool-a".to_string(),
95 + arguments: "{}".to_string(),
96 + }),
97 + AssistantContentBlock::ToolCall(AssistantToolCall {
98 + id: "call-b".to_string(),
99 + name: "tool-b".to_string(),
100 + arguments: "{}".to_string(),
101 + }),
102 + ]),
103 + ChatMessage::tool_response(
104 + vec![ChatContentPart::text("result-b"), image_part("image-b")],
105 + "call-b",
106 + ),
107 + ChatMessage::tool_response(
108 + vec![ChatContentPart::text("result-a"), image_part("image-a")],
109 + "call-a",
110 + ),
111 + ],
112 + ..ChatRequest::for_test()
113 + };
114 +
115 + let rendered = renderer.render(&request).unwrap();
116 + let media_order = rendered.media_order.unwrap();
117 + let prompt = rendered.prompt.into_text().unwrap();
118 +
119 + assert!(prompt.find("result-a").unwrap() < prompt.find("result-b").unwrap());
120 + expect![[r#"
121 + [
122 + MediaPartSource {
123 + message_index: 2,
124 + content_part_index: 1,
125 + },
126 + MediaPartSource {
127 + message_index: 1,
128 + content_part_index: 1,
129 + },
130 + ]
131 + "#]]
132 + .assert_debug_eq(&media_order);
133 +}
134 +
135 +#[test]
136 +fn media_order_excludes_dropped_historical_developer_content() {
137 + let renderer = DeepSeekV41ChatRenderer::new();
138 + let request = ChatRequest {
139 + messages: vec![
140 + ChatMessage::developer(vec![image_part("historical-image")], None),
141 + ChatMessage::user("question"),
142 + ],
143 + ..ChatRequest::for_test()
144 + };
145 +
146 + let rendered = renderer.render(&request).unwrap();
147 + let media_order = rendered.media_order.unwrap();
148 + let prompt = rendered.prompt.into_text().unwrap();
149 +
150 + assert!(!prompt.contains("<｜deepseek_image｜>"));
151 + assert!(media_order.is_empty());
152 +}
153 +
154 +fn image_part(uuid: &str) -> ChatContentPart {
155 + ChatContentPart::ImageUrl {
156 + image_url: format!("data:image/png;base64,{uuid}"),
157 + detail: None,
158 + uuid: Some(uuid.to_string()),
159 + }
160 +}
161 +
162 #[test]
163 fn maps_reasoning_effort_to_reference_numeric_budget() {
164 for (effort, budget) in [
rust/src/chat/src/renderer/harmony/mod.rs
+1
@@ -116,6 +116,7 @@ impl ChatRenderer for HarmonyChatRenderer {
116 fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
117 Ok(RenderedPrompt {
118 prompt: Prompt::TokenIds(self.render_token_ids(request)?),
119 + media_order: None,
120 effective_template_kwargs: request_template_kwargs(request),
121 })
122 }
rust/src/chat/src/renderer/hf/mod.rs
+1
@@ -217,6 +217,7 @@ impl HfChatRenderer {
217
218 Ok(RenderedPrompt {
219 prompt: Prompt::Text(prompt),
220 + media_order: None,
221 effective_template_kwargs,
222 })
223 }
rust/src/chat/src/renderer/inkling/mod.rs
+1
@@ -346,6 +346,7 @@ impl ChatRenderer for InklingChatRenderer {
346
347 Ok(RenderedPrompt {
348 prompt: Prompt::TokenIds(out),
349 + media_order: None,
350 effective_template_kwargs,
351 })
352 }
rust/src/chat/src/renderer/kimi_k3/encoding.rs
+37 -14
@@ -10,6 +10,7 @@ use std::collections::HashMap;
10 use serde_json::{Map, Value, json};
11 use vllm_tokenizer::Tokenizer;
12
13 +use super::super::MediaPartSource;
14 use crate::error::{Error, Result};
15 use crate::request::{
16 ChatContent, ChatContentPart, ChatMessage, ChatRequest, ChatTool, ChatToolChoice,
@@ -29,6 +30,7 @@ const VALID_THINKING_EFFORTS: &[&str] = &["low", "high", "max"];
30 pub(super) struct K3TokenWriter<'a> {
31 tokenizer: &'a dyn Tokenizer,
32 token_ids: Vec<u32>,
33 + media_order: Vec<MediaPartSource>,
34 }
35
36 impl<'a> K3TokenWriter<'a> {
@@ -36,6 +38,7 @@ impl<'a> K3TokenWriter<'a> {
38 Self {
39 tokenizer,
40 token_ids: Vec::new(),
41 + media_order: Vec::new(),
42 }
43 }
44
@@ -55,13 +58,25 @@ impl<'a> K3TokenWriter<'a> {
58 Ok(())
59 }
60
58 - pub(super) fn finish(self) -> Vec<u32> {
59 - self.token_ids
61 + fn image(&mut self, message_index: usize, content_part_index: usize) -> Result<()> {
62 + self.control(IMAGE_PLACEHOLDER)?;
63 + self.media_order.push(MediaPartSource {
64 + message_index,
65 + content_part_index,
66 + });
67 + Ok(())
68 + }
69 +
70 + pub(super) fn finish(self) -> (Vec<u32>, Vec<MediaPartSource>) {
71 + (self.token_ids, self.media_order)
72 }
73 }
74
63 -/// Render and tokenize one chat request using K3's segment-aware contract.
64 -pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -> Result<Vec<u32>> {
75 +/// Render and tokenize one chat request while recording media in placeholder order.
76 +pub(super) fn render_request_with_media_order(
77 + request: &ChatRequest,
78 + tokenizer: &dyn Tokenizer,
79 +) -> Result<(Vec<u32>, Vec<MediaPartSource>)> {
80 let thinking = thinking_enabled(request)?;
81 let thinking_effort = thinking.then(|| thinking_effort(request)).transpose()?;
82 let tools = request_tools(request);
@@ -126,7 +141,7 @@ pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -
141 resolved.sort_by_key(|item| (item.0, item.1));
142 }
143
129 - for (position, _, content, name) in resolved {
144 + for (position, message_index, content, name) in resolved {
145 let tool_name = name.as_deref().ok_or_else(|| {
146 Error::ChatTemplate(
147 "Kimi K3 tool messages need a resolvable tool name: \
@@ -135,7 +150,7 @@ pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -
150 .to_string(),
151 )
152 })?;
138 - write_tool_message(out, tool_name, position, &content)?;
153 + write_tool_message(out, tool_name, position, message_index, &content)?;
154 }
155 Ok(())
156 };
@@ -161,14 +176,14 @@ pub(super) fn render_request(request: &ChatRequest, tokenizer: &dyn Tokenizer) -
176 } if !local_tools.is_empty() => {
177 write_tool_declare(&mut out, local_tools, true)?;
178 if !content_is_empty(content) {
164 - write_role_message(&mut out, "system", None, content)?;
179 + write_role_message(&mut out, "system", None, message_index, content)?;
180 }
181 }
182 ChatMessage::System { content } | ChatMessage::Developer { content, .. } => {
168 - write_role_message(&mut out, "system", None, content)?;
183 + write_role_message(&mut out, "system", None, message_index, content)?;
184 }
185 ChatMessage::User { content } => {
171 - write_role_message(&mut out, "user", None, content)?;
186 + write_role_message(&mut out, "user", None, message_index, content)?;
187 }
188 ChatMessage::Assistant { content } => {
189 tool_call_id_index.clear();
@@ -364,6 +379,7 @@ fn write_role_message(
379 out: &mut K3TokenWriter<'_>,
380 role: &str,
381 name: Option<&str>,
382 + message_index: usize,
383 content: &ChatContent,
384 ) -> Result<()> {
385 let mut attrs = vec![("role", role.to_string())];
@@ -372,7 +388,7 @@ fn write_role_message(
388 }
389 let attr_refs: Vec<(&str, &str)> = attrs.iter().map(|(k, v)| (*k, v.as_str())).collect();
390 write_open_tag(out, "message", &attr_refs)?;
375 - write_content(out, content)?;
391 + write_content(out, message_index, content)?;
392 write_close_tag(out, "message")?;
393 out.control(END_OF_MSG)
394 }
@@ -381,6 +397,7 @@ fn write_tool_message(
397 out: &mut K3TokenWriter<'_>,
398 tool_name: &str,
399 index: usize,
400 + message_index: usize,
401 content: &ChatContent,
402 ) -> Result<()> {
403 let index_str = index.to_string();
@@ -389,7 +406,7 @@ fn write_tool_message(
406 "message",
407 &[("role", "tool"), ("tool", tool_name), ("index", &index_str)],
408 )?;
392 - write_content(out, content)?;
409 + write_content(out, message_index, content)?;
410 write_close_tag(out, "message")?;
411 out.control(END_OF_MSG)
412 }
@@ -471,14 +488,20 @@ fn write_assistant_tool_call(
488 write_close_tag(out, "call")
489 }
490
474 -fn write_content(out: &mut K3TokenWriter<'_>, content: &ChatContent) -> Result<()> {
491 +fn write_content(
492 + out: &mut K3TokenWriter<'_>,
493 + message_index: usize,
494 + content: &ChatContent,
495 +) -> Result<()> {
496 match content {
497 ChatContent::Text(text) => write_text_with_images(out, text),
498 ChatContent::Parts(parts) => {
478 - for part in parts {
499 + for (content_part_index, part) in parts.iter().enumerate() {
500 match part {
501 ChatContentPart::Text { text } => write_text_with_images(out, text)?,
481 - ChatContentPart::ImageUrl { .. } => out.control(IMAGE_PLACEHOLDER)?,
502 + ChatContentPart::ImageUrl { .. } => {
503 + out.image(message_index, content_part_index)?;
504 + }
505 ChatContentPart::VideoUrl { .. } => {
506 return Err(Error::UnsupportedMultimodalContent("video_url"));
507 }
rust/src/chat/src/renderer/kimi_k3/mod.rs
+4 -1
@@ -30,9 +30,12 @@ impl KimiK3ChatRenderer {
30 impl ChatRenderer for KimiK3ChatRenderer {
31 fn render(&self, request: &ChatRequest) -> Result<RenderedPrompt> {
32 request.validate()?;
33 + let (token_ids, media_order) =
34 + encoding::render_request_with_media_order(request, self.tokenizer.as_ref())?;
35
36 Ok(RenderedPrompt {
35 - prompt: Prompt::TokenIds(encoding::render_request(request, self.tokenizer.as_ref())?),
37 + prompt: Prompt::TokenIds(token_ids),
38 + media_order: Some(media_order),
39 effective_template_kwargs: request_template_kwargs(request),
40 })
41 }
rust/src/chat/src/renderer/kimi_k3/tests.rs
+59
@@ -284,6 +284,65 @@ fn partial_tool_results_keep_assistant_call_position() {
284 );
285 }
286
287 +#[test]
288 +fn media_order_follows_reordered_tool_results() {
289 + let mut request = crate::request::ChatRequest::for_test();
290 + request.messages = vec![
291 + ChatMessage::assistant_blocks(vec![
292 + AssistantContentBlock::ToolCall(AssistantToolCall {
293 + id: "call-a".to_string(),
294 + name: "tool-a".to_string(),
295 + arguments: "{}".to_string(),
296 + }),
297 + AssistantContentBlock::ToolCall(AssistantToolCall {
298 + id: "call-b".to_string(),
299 + name: "tool-b".to_string(),
300 + arguments: "{}".to_string(),
301 + }),
302 + ]),
303 + ChatMessage::tool_response(
304 + vec![ChatContentPart::text("result-b"), image_part("image-b")],
305 + "call-b",
306 + ),
307 + ChatMessage::tool_response(
308 + vec![ChatContentPart::text("result-a"), image_part("image-a")],
309 + "call-a",
310 + ),
311 + ];
312 + request.chat_options.generation_prompt_mode = GenerationPromptMode::NoGenerationPrompt;
313 + let renderer = KimiK3ChatRenderer::new(Arc::new(test_tokenizer()));
314 +
315 + let rendered = renderer.render(&request).unwrap();
316 + let media_order = rendered.media_order.unwrap();
317 + let Prompt::TokenIds(token_ids) = rendered.prompt else {
318 + panic!("kimi k3 renderer should return token IDs")
319 + };
320 + let prompt = test_tokenizer().decode(&token_ids, false).unwrap();
321 +
322 + assert!(prompt.find("result-a").unwrap() < prompt.find("result-b").unwrap());
323 + expect![[r#"
324 + [
325 + MediaPartSource {
326 + message_index: 2,
327 + content_part_index: 1,
328 + },
329 + MediaPartSource {
330 + message_index: 1,
331 + content_part_index: 1,
332 + },
333 + ]
334 + "#]]
335 + .assert_debug_eq(&media_order);
336 +}
337 +
338 +fn image_part(uuid: &str) -> ChatContentPart {
339 + ChatContentPart::ImageUrl {
340 + image_url: format!("data:image/png;base64,{uuid}"),
341 + detail: None,
342 + uuid: Some(uuid.to_string()),
343 + }
344 +}
345 +
346 #[test]
347 fn defaults_thinking_effort_to_max() {
348 let rendered = render_request(&crate::request::ChatRequest::for_test());
rust/src/chat/src/renderer/mod.rs
+13
@@ -30,11 +30,24 @@ pub use inkling::InklingChatRenderer;
30 pub use kimi_k3::KimiK3ChatRenderer;
31 pub use selection::RendererSelection;
32
33 +/// Location of one multimodal content part in the source chat request.
34 +#[derive(Debug, Clone, Copy, PartialEq, Eq)]
35 +pub struct MediaPartSource {
36 + /// Zero-based message index in the source request.
37 + pub message_index: usize,
38 + /// Zero-based content-part index within the source message.
39 + pub content_part_index: usize,
40 +}
41 +
42 /// Rendered chat prompt submitted to the text backend.
43 #[derive(Debug, Clone, PartialEq)]
44 pub struct RenderedPrompt {
45 /// The rendered prompt, either as text or already tokenized.
46 pub prompt: Prompt,
47 + /// Media sources in rendered placeholder order.
48 + /// `None` uses the message and content-part order in the request (e.g. Jinja).
49 + /// `Some` uses exactly the listed sources; an empty list omits all media.
50 + pub media_order: Option<Vec<MediaPartSource>>,
51 /// Effective chat-template kwargs visible to the renderer after applying
52 /// server defaults, request overrides, and typed reasoning controls.
53 pub effective_template_kwargs: HashMap<String, Value>,
rust/src/chat/tests/chat.rs
+1
@@ -250,6 +250,7 @@ impl ChatRenderer for FakeChatBackend {
250
251 Ok(RenderedPrompt {
252 prompt: Prompt::Text(prompt),
253 + media_order: None,
254 effective_template_kwargs: request.chat_options.template_kwargs.clone(),
255 })
256 }
rust/src/server/src/grpc/tests.rs
+1
@@ -252,6 +252,7 @@ impl ChatRenderer for FakeTextBackend {
252 fn render(&self, _request: &ChatRequest) -> vllm_chat::Result<RenderedPrompt> {
253 Ok(RenderedPrompt {
254 prompt: Prompt::Text(String::new()),
255 + media_order: None,
256 effective_template_kwargs: Default::default(),
257 })
258 }
rust/src/server/src/routes/http_client_tests.rs
+1
@@ -186,6 +186,7 @@ impl ChatRenderer for FakeChatBackend {
186 }
187 Ok(RenderedPrompt {
188 prompt: Prompt::Text(prompt),
189 + media_order: None,
190 effective_template_kwargs: Default::default(),
191 })
192 }
rust/src/server/src/routes/tests.rs
+1
@@ -540,6 +540,7 @@ impl ChatRenderer for FakeChatBackend {
540 }
541 Ok(vllm_chat::RenderedPrompt {
542 prompt: Prompt::Text(prompt),
543 + media_order: None,
544 effective_template_kwargs: Default::default(),
545 })
546 }