@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 074bdd0d99

[Rust Frontend] Integrate MM video support (#47959)

Signed-off-by: Bugen Zhao <i@bugenzhao.com>

Bugen Zhao committed Jul 10, 2026 at 16:15 UTC 074bdd0d9961fb7f81c9311e00c00cfd5aff77a3
16 files changed +1571 -516
rust/Cargo.lock
+3 -1
@@ -2167,7 +2167,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
2167 [[package]]
2168 name = "llm-multimodal"
2169 version = "1.7.1"
2170 -source = "git+https://github.com/smg-project/llm-multimodal?rev=7d74582aeaf0e4086a44964382655d22f1af0686#7d74582aeaf0e4086a44964382655d22f1af0686"
2170 +source = "git+https://github.com/smg-project/llm-multimodal?rev=c8a29dcc755139fdc26185f400ea48c6d6d48273#c8a29dcc755139fdc26185f400ea48c6d6d48273"
2171 dependencies = [
2172 "anyhow",
2173 "base64 0.22.1",
@@ -2473,6 +2473,7 @@ dependencies = [
2473 "portable-atomic",
2474 "portable-atomic-util",
2475 "rawpointer",
2476 + "serde",
2477 ]
2478
2479 [[package]]
@@ -5093,6 +5094,7 @@ dependencies = [
5094 "llm-multimodal",
5095 "minijinja",
5096 "minijinja-contrib",
5097 + "ndarray 0.17.2",
5098 "oss-harmony",
5099 "paste",
5100 "reqwest",
rust/Cargo.toml
+2 -2
@@ -53,12 +53,12 @@ hyper-util = { version = "0.1.20", features = [
53 indexmap = "2.13.0"
54 itertools = "0.14.0"
55 libc = "0.2.177"
56 -llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "7d74582aeaf0e4086a44964382655d22f1af0686" }
56 +llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "c8a29dcc755139fdc26185f400ea48c6d6d48273" }
57 mimalloc = "0.1.52"
58 minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
59 minijinja-contrib = { version = "2.0", features = ["pycompat"] }
60 native-tls-vendored = { package = "native-tls", version = "0.2.18", features = ["vendored"] }
61 -ndarray = { version = "0.16.1", features = ["serde"] }
61 +ndarray = { version = "0.17", features = ["serde"] }
62 openai-harmony = { package = "oss-harmony", git = "https://github.com/oss-harmony/harmony", tag = "v0.0.11", default-features = false }
63 openai-protocol = "1.6.0"
64 openssl = "0.10"
rust/src/chat/Cargo.toml
+1
@@ -42,6 +42,7 @@ anyhow.workspace = true
42 bytes.workspace = true
43 clap.workspace = true
44 expect-test.workspace = true
45 +ndarray.workspace = true
46 paste.workspace = true
47 rmp-serde.workspace = true
48 serial_test.workspace = true
rust/src/chat/src/backend/hf.rs
+13 -4
@@ -10,7 +10,7 @@ use crate::backend::{
10 NewChatOutputProcessorOptions,
11 };
12 use crate::error::Result;
13 -use crate::multimodal::MultimodalModelInfo;
13 +use crate::multimodal::{MultimodalConfigFiles, MultimodalModelInfo};
14 use crate::output::{
15 DefaultChatOutputProcessor, HarmonyChatOutputProcessor, validate_harmony_parser_overrides,
16 };
@@ -46,8 +46,12 @@ impl HfChatBackend {
46 MultimodalModelInfo::from_paths(
47 model_id.clone(),
48 (!model_type.is_empty()).then_some(model_type.to_string()),
49 - files.config_path.as_deref(),
50 - files.preprocessor_config_path.as_deref(),
49 + MultimodalConfigFiles {
50 + config: files.config_path.as_deref(),
51 + preprocessor_config: files.preprocessor_config_path.as_deref(),
52 + video_preprocessor_config: files.video_preprocessor_config_path.as_deref(),
53 + processor_config: files.processor_config_path.as_deref(),
54 + },
55 tokenizer.clone(),
56 )?
57 };
@@ -139,8 +143,11 @@ pub(super) async fn load_model_backends(
143 fn resolve_multimodal_render_info(
144 info: Option<&MultimodalModelInfo>,
145 ) -> Option<MultimodalRenderInfo> {
146 + use llm_multimodal::Modality;
147 +
148 info.map(|info| MultimodalRenderInfo {
143 - placeholder_token: info.placeholder_token().to_string(),
149 + image_token: info.placeholder_token(Modality::Image).map(str::to_string),
150 + video_token: info.placeholder_token(Modality::Video).map(str::to_string),
151 })
152 }
153
@@ -192,6 +199,8 @@ mod tests {
199 tokenizer_config_path: Some(tokenizer_config_path),
200 generation_config_path: None,
201 preprocessor_config_path: None,
202 + video_preprocessor_config_path: None,
203 + processor_config_path: None,
204 chat_template_path: None,
205 config_path: Some(config_path),
206 }
rust/src/chat/src/error.rs
+31 -1
@@ -1,5 +1,5 @@
1 use thiserror::Error;
2 -use thiserror_ext::Macro;
2 +use thiserror_ext::{AsReport as _, Macro};
3
4 type BoxedError = Box<dyn std::error::Error + Send + Sync>;
5
@@ -18,6 +18,8 @@ pub enum Error {
18 UnsupportedMultimodalRenderer,
19 #[error("unsupported multimodal content: {0}")]
20 UnsupportedMultimodalContent(&'static str),
21 + #[error("`{modality}` input is not supported by this model")]
22 + UnsupportedModality { modality: String },
23 #[error("multimodal preprocessing error: {0}")]
24 Multimodal(#[message] String),
25 #[error("{kind} parsing is not available for model `{model_id}`")]
@@ -80,11 +82,39 @@ impl Error {
82 match self {
83 Self::PromptTooLong { .. } => true,
84 Self::Text(error) => error.is_request_validation_error(),
85 + Self::UnsupportedMultimodalRenderer
86 + | Self::UnsupportedMultimodalContent(_)
87 + | Self::UnsupportedModality { .. } => true,
88 +
89 _ => false,
90 }
91 }
92 }
93
94 +impl From<llm_multimodal::MediaConnectorError> for Error {
95 + fn from(error: llm_multimodal::MediaConnectorError) -> Self {
96 + Self::Multimodal(error.to_report_string())
97 + }
98 +}
99 +
100 +impl From<llm_multimodal::MultiModalError> for Error {
101 + fn from(error: llm_multimodal::MultiModalError) -> Self {
102 + Self::Multimodal(error.to_report_string())
103 + }
104 +}
105 +
106 +impl From<llm_multimodal::TransformError> for Error {
107 + fn from(error: llm_multimodal::TransformError) -> Self {
108 + Self::Multimodal(error.to_report_string())
109 + }
110 +}
111 +
112 +impl From<llm_multimodal::registry::ModelRegistryError> for Error {
113 + fn from(error: llm_multimodal::registry::ModelRegistryError) -> Self {
114 + Self::Multimodal(error.to_report_string())
115 + }
116 +}
117 +
118 /// Format the available-parser suffix used in user-facing error messages.
119 fn available_parser_hint(available_names: &[String]) -> String {
120 if available_names.is_empty() {
rust/src/chat/src/multimodal.rs
+420 -474
@@ -1,8 +1,8 @@
1 -//! Chat-layer multimodal image preparation.
1 +//! Chat-layer multimodal media preparation.
2 //!
3 -//! This module owns the narrow image-only multimodal path for chat requests:
4 -//! it extracts image parts from structured chat messages, fetches and
5 -//! preprocesses them through `llm-multimodal`, expands rendered prompt
3 +//! This module owns the multimodal path for chat requests: it extracts media
4 +//! parts from structured chat messages, fetches and preprocesses them through
5 +//! `llm-multimodal` one modality at a time, expands rendered prompt
6 //! placeholders after tokenization, and builds the engine-facing
7 //! `MmFeatures` payload.
8 //!
@@ -16,19 +16,15 @@ use std::sync::{Arc, LazyLock};
16
17 use itertools::izip;
18 use llm_multimodal::{
19 - AsyncMultiModalTracker, FieldLayout, MediaConnector, MediaConnectorConfig, MediaContentPart,
20 - Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry, PreProcessorConfig,
21 - PreprocessedEncoderInputs as PreprocessedImages, PromptReplacement, Tokenizer as TokenResolver,
22 - TrackedMedia, VisionPreProcessor as ImagePreProcessor,
23 - VisionProcessorRegistry as ImageProcessorRegistry,
19 + AsyncMultiModalTracker, FieldLayout, ImageFrame, MediaConnector, MediaConnectorConfig,
20 + MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry,
21 + PreProcessorConfig, PreprocessedEncoderInputs, PromptReplacement, Tokenizer as TokenResolver,
22 + TrackedMedia, VideoClip, VisionPreProcessor, VisionProcessorRegistry,
23 };
24 +use thiserror_ext::AsReport as _;
25 use tracing::warn;
26 use vllm_engine_core_client::protocol::dtype::ModelDtype;
27 -use vllm_engine_core_client::protocol::multimodal::{
28 - MmBatchedField, MmFeatureSpec, MmFeatures, MmField, MmFieldElem, MmFlatField, MmKwargsItem,
29 - MmSharedField, MmSlice, PlaceholderRange, SliceSpec,
30 -};
31 -use vllm_engine_core_client::protocol::tensor::WireTensor;
27 +use vllm_engine_core_client::protocol::multimodal::{MmFeatureSpec, MmFeatures, MmKwargsItem};
28 use vllm_text::Prompt;
29 use vllm_text::tokenizer::{DynTokenizer, Tokenizer};
30
@@ -36,14 +32,20 @@ use crate::error::{Error, Result, bail_multimodal, multimodal};
32 use crate::renderer::RenderedPrompt;
33 use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
34
35 +mod expand;
36 +mod image;
37 mod tensor;
38 +mod video;
39 +
40 +use self::expand::expand_prompt_token_ids;
41
42 /// Resolved multimodal support for one loaded model.
43 #[derive(Clone)]
44 pub struct MultimodalModelInfo {
45 context: MultimodalModelContext,
46 spec: ResolvedMultimodalSpec,
46 - image_processor: ResolvedImageProcessor,
47 + image: Option<ModalitySupport>,
48 + video: Option<ModalitySupport>,
49 media_connector: Arc<MediaConnector>,
50 }
51
@@ -75,91 +77,171 @@ impl MultimodalModelContext {
77 REGISTRY.lookup(&self.metadata())
78 }
79
78 - /// Resolve a static image preprocessor for one loaded model.
79 - fn resolve_image_processor(&self) -> Option<&'static dyn ImagePreProcessor> {
80 - static REGISTRY: LazyLock<ImageProcessorRegistry> =
81 - LazyLock::new(ImageProcessorRegistry::with_defaults);
80 + /// Resolve a static vision preprocessor for one loaded model.
81 + ///
82 + /// The vision preprocessor serves both the image and video modalities.
83 + fn resolve_vision_processor(&self) -> Option<&'static dyn VisionPreProcessor> {
84 + static REGISTRY: LazyLock<VisionProcessorRegistry> =
85 + LazyLock::new(VisionProcessorRegistry::with_defaults);
86 REGISTRY.find(&self.model_id, self.model_type.as_deref())
87 }
88 }
89
86 -/// Static model-specific prompt and tensor-layout behavior.
90 +/// Static model-specific tensor-layout behavior shared across modalities.
91 #[derive(Clone)]
92 struct ResolvedMultimodalSpec {
93 raw: &'static dyn ModelProcessorSpec,
90 - placeholder_token: String,
91 - placeholder_marker_token_id: u32,
92 - placeholder_embed_token_id: u32,
94 field_layouts: HashMap<String, FieldLayout>,
95 keep_on_cpu_keys: HashSet<String>,
96 }
97
98 impl ResolvedMultimodalSpec {
98 - fn new(raw: &'static dyn ModelProcessorSpec, context: &MultimodalModelContext) -> Result<Self> {
99 - let metadata = context.metadata();
100 - let placeholder_token =
101 - raw.placeholder_token(&metadata).map_err(|error| multimodal!("{error}"))?;
102 - // This is the rendered prompt marker, so resolve it from the token
103 - // string itself. Do not use `ModelProcessorSpec::placeholder_token_id()`:
104 - // for some specs that ID is the replacement vision/patch token,
105 - // not necessarily the token ID of `placeholder_token`.
106 - let placeholder_marker_token_id =
107 - context.tokenizer().token_to_id(&placeholder_token).ok_or_else(|| {
108 - multimodal!(
109 - "placeholder token `{placeholder_token}` is not in the tokenizer vocabulary"
110 - )
111 - })?;
112 - let placeholder_embed_token_id =
113 - raw.placeholder_token_id(&metadata).map_err(|error| multimodal!("{error}"))? as u32;
114 -
115 - Ok(Self {
99 + fn new(raw: &'static dyn ModelProcessorSpec) -> Self {
100 + Self {
101 raw,
117 - placeholder_token,
118 - placeholder_marker_token_id,
119 - placeholder_embed_token_id,
102 field_layouts: raw.field_layouts(),
103 keep_on_cpu_keys: raw.keep_on_cpu_keys().into_iter().collect(),
122 - })
104 + }
105 }
106
125 - fn prompt_replacements(
107 + fn prompt_replacements_for(
108 &self,
109 context: &MultimodalModelContext,
128 - preprocessed: &PreprocessedImages,
110 + preprocessed: &PreprocessedEncoderInputs,
111 + modality: Modality,
112 ) -> Result<Vec<PromptReplacement>> {
130 - self.raw
131 - .prompt_replacements(&context.metadata(), preprocessed)
132 - .map_err(|error| multimodal!("{error}"))
113 + Ok(self.raw.prompt_replacements_for(&context.metadata(), preprocessed, modality)?)
114 }
115 }
116
136 -/// Static image preprocessor plus its loaded config.
117 +/// Resolved placeholder tokens for one modality.
118 #[derive(Clone)]
138 -struct ResolvedImageProcessor {
139 - raw: &'static dyn ImagePreProcessor,
119 +struct ResolvedPlaceholder {
120 + token: String,
121 + /// The token ID emitted for `token` in the rendered prompt.
122 + marker_token_id: u32,
123 + /// The model-declared embed token ID marked in `is_embed` masks.
124 + embed_token_id: u32,
125 +}
126 +
127 +impl ResolvedPlaceholder {
128 + fn resolve(
129 + raw: &'static dyn ModelProcessorSpec,
130 + context: &MultimodalModelContext,
131 + modality: Modality,
132 + ) -> Result<Self> {
133 + let metadata = context.metadata();
134 + let token = raw.placeholder_token_for(&metadata, modality)?;
135 + // This is the rendered prompt marker, so resolve it from the token
136 + // string itself. Do not use `ModelProcessorSpec::placeholder_token_id_for()`:
137 + // for some specs that ID is the replacement vision/patch token,
138 + // not necessarily the token ID of the placeholder token.
139 + let marker_token_id = context.tokenizer().token_to_id(&token).ok_or_else(|| {
140 + multimodal!("placeholder token `{token}` is not in the tokenizer vocabulary")
141 + })?;
142 + let embed_token_id = raw.placeholder_token_id_for(&metadata, modality)? as u32;
143 +
144 + Ok(Self {
145 + token,
146 + marker_token_id,
147 + embed_token_id,
148 + })
149 + }
150 +}
151 +
152 +/// Static per-modality vision preprocessor plus its loaded config and
153 +/// resolved placeholder tokens.
154 +#[derive(Clone)]
155 +struct ModalitySupport {
156 + placeholder: ResolvedPlaceholder,
157 + processor: &'static dyn VisionPreProcessor,
158 config: PreProcessorConfig,
159 }
160
143 -/// Request-scoped fetched media, kept together with tracker UUID metadata.
144 -struct FetchedImageMedia {
145 - frames: Vec<Arc<llm_multimodal::ImageFrame>>,
146 - uuids: Vec<Option<String>>,
161 +/// Model-repo config file locations consumed by multimodal support.
162 +#[derive(Debug, Default, Clone, Copy)]
163 +pub struct MultimodalConfigFiles<'a> {
164 + pub config: Option<&'a Path>,
165 + pub preprocessor_config: Option<&'a Path>,
166 + /// Video-specific preprocessor config (`video_preprocessor_config.json`).
167 + pub video_preprocessor_config: Option<&'a Path>,
168 + /// Combined processor config (`processor_config.json`), whose modality
169 + /// sections are fallback preprocessor config sources.
170 + pub processor_config: Option<&'a Path>,
171 +}
172 +
173 +/// Load a modality's dedicated preprocessor config, falling back to its section
174 +/// in the combined processor config.
175 +fn load_preprocessor_config(
176 + dedicated_path: Option<&Path>,
177 + dedicated_name: &str,
178 + processor_config_path: Option<&Path>,
179 + processor_section: &str,
180 +) -> Result<Option<PreProcessorConfig>> {
181 + if let Some(path) = dedicated_path {
182 + let text = fs::read_to_string(path)
183 + .map_err(|error| multimodal!("failed to read {dedicated_name}: {error}"))?;
184 + let config = PreProcessorConfig::from_json(&text)
185 + .map_err(|error| multimodal!("failed to parse {dedicated_name}: {error}"))?;
186 + return Ok(Some(config));
187 + }
188 +
189 + let Some(path) = processor_config_path else {
190 + return Ok(None);
191 + };
192 + let text = fs::read_to_string(path)
193 + .map_err(|error| multimodal!("failed to read processor_config.json: {error}"))?;
194 + let value: serde_json::Value = serde_json::from_str(&text)
195 + .map_err(|error| multimodal!("failed to parse processor_config.json: {error}"))?;
196 + let Some(processor) = value.get(processor_section) else {
197 + return Ok(None);
198 + };
199 + let config = PreProcessorConfig::from_value(processor.clone()).map_err(|error| {
200 + multimodal!("failed to parse {processor_section} from processor_config.json: {error}")
201 + })?;
202 + Ok(Some(config))
203 +}
204 +
205 +/// Request-scoped fetched media, split per modality with tracker UUID
206 +/// metadata preserved in request order.
207 +struct FetchedMedia {
208 + images: Vec<Arc<ImageFrame>>,
209 + image_uuids: Vec<Option<String>>,
210 + videos: Vec<Arc<VideoClip>>,
211 + video_uuids: Vec<Option<String>>,
212 +}
213 +
214 +/// One modality's preprocessed output, ready for the shared expansion and
215 +/// feature-assembly tail.
216 +struct PreparedMedia {
217 + modality: Modality,
218 + placeholder: ResolvedPlaceholder,
219 + /// One replacement per media item, in request order.
220 + replacements: Vec<PromptReplacement>,
221 + /// One entry per media item, aligned with `replacements`.
222 + items: Vec<PreparedItem>,
223 +}
224 +
225 +/// One media item's complete engine kwargs plus identity metadata.
226 +struct PreparedItem {
227 + data: MmKwargsItem,
228 + hash: String,
229 + uuid: Option<String>,
230 }
231
232 impl MultimodalModelInfo {
233 /// Load and resolve multimodal support from model files.
234 ///
152 - /// Returns `Ok(Some(_))` only when both the model spec and image processor
153 - /// are registered. File read/parse failures are real errors; unsupported
154 - /// model families are logged and returned as `Ok(None)`.
235 + /// Returns `Ok(Some(_))` only when the model spec is registered and at
236 + /// least one modality resolves. File read/parse failures are real errors;
237 + /// unsupported model families are logged and returned as `Ok(None)`.
238 pub fn from_paths(
239 model_id: String,
240 model_type: Option<String>,
158 - config_path: Option<&Path>,
159 - preprocessor_config_path: Option<&Path>,
241 + files: MultimodalConfigFiles<'_>,
242 tokenizer: DynTokenizer,
243 ) -> Result<Option<Self>> {
162 - let config = match config_path {
244 + let config = match files.config {
245 Some(path) => {
246 let text = fs::read_to_string(path)
247 .map_err(|error| multimodal!("failed to read config.json: {error}"))?;
@@ -168,17 +250,20 @@ impl MultimodalModelInfo {
250 }
251 None => serde_json::Value::Object(Default::default()),
252 };
171 - let preprocessor_config = match preprocessor_config_path {
172 - Some(path) => {
173 - let text = fs::read_to_string(path).map_err(|error| {
174 - multimodal!("failed to read preprocessor_config.json: {error}")
175 - })?;
176 - PreProcessorConfig::from_json(&text).map_err(|error| {
177 - multimodal!("failed to parse preprocessor_config.json: {error}")
178 - })?
179 - }
180 - None => PreProcessorConfig::default(),
181 - };
253 + let image_preprocessor_config = load_preprocessor_config(
254 + files.preprocessor_config,
255 + "preprocessor_config.json",
256 + files.processor_config,
257 + "image_processor",
258 + )?
259 + .unwrap_or_default();
260 + let video_preprocessor_config = load_preprocessor_config(
261 + files.video_preprocessor_config,
262 + "video_preprocessor_config.json",
263 + files.processor_config,
264 + "video_processor",
265 + )?
266 + .unwrap_or_else(|| image_preprocessor_config.clone());
267
268 let context = MultimodalModelContext {
269 model_id,
@@ -187,7 +272,21 @@ impl MultimodalModelInfo {
272 tokenizer: TokenizerResolver(tokenizer),
273 };
274
190 - let Some(spec) = context.resolve_model_spec() else {
275 + Self::from_loaded(
276 + context,
277 + image_preprocessor_config,
278 + video_preprocessor_config,
279 + )
280 + }
281 +
282 + /// Resolve multimodal support from an assembled context and parsed
283 + /// preprocessor configs.
284 + fn from_loaded(
285 + context: MultimodalModelContext,
286 + image_preprocessor_config: PreProcessorConfig,
287 + video_preprocessor_config: PreProcessorConfig,
288 + ) -> Result<Option<Self>> {
289 + let Some(raw_spec) = context.resolve_model_spec() else {
290 warn!(
291 model_id = context.model_id,
292 model_type = context.model_type,
@@ -195,47 +294,99 @@ impl MultimodalModelInfo {
294 );
295 return Ok(None);
296 };
198 - let spec = ResolvedMultimodalSpec::new(spec, &context)?;
297
200 - let Some(image_processor) = context.resolve_image_processor() else {
298 + let Some(processor) = context.resolve_vision_processor() else {
299 warn!(
300 model_id = context.model_id,
301 model_type = context.model_type,
204 - "image processor is not registered; disabling multimodal support for this model"
302 + "vision processor is not registered; disabling multimodal support for this model"
303 );
304 return Ok(None);
305 };
306
209 - let media_connector = Arc::new(
210 - MediaConnector::new(reqwest::Client::new(), MediaConnectorConfig::default())
211 - .map_err(|error| multimodal!("{error}"))?,
212 - );
307 + // Warn and disable the modality if the placeholder resolution fails.
308 + let resolve_placeholder =
309 + |modality: Modality| match ResolvedPlaceholder::resolve(raw_spec, &context, modality) {
310 + Ok(placeholder) => Some(placeholder),
311 + Err(error) => {
312 + warn!(
313 + model_id = context.model_id,
314 + %modality,
315 + error = %error.as_report(),
316 + "placeholder tokens did not resolve; disabling this modality for this model"
317 + );
318 + None
319 + }
320 + };
321 +
322 + let image = resolve_placeholder(Modality::Image).map(|placeholder| ModalitySupport {
323 + placeholder,
324 + processor,
325 + config: image_preprocessor_config,
326 + });
327 +
328 + let video = resolve_placeholder(Modality::Video).and_then(|placeholder| {
329 + // Placeholder expansion attributes markers to modalities by token
330 + // ID, so a marker shared with the image modality is ambiguous.
331 + let image_marker = image.as_ref().map(|image| image.placeholder.marker_token_id);
332 + if image_marker == Some(placeholder.marker_token_id) {
333 + warn!(
334 + model_id = context.model_id,
335 + token = placeholder.token,
336 + "video placeholder token collides with the image placeholder; disabling video support for this model"
337 + );
338 + None
339 + } else {
340 + Some(ModalitySupport {
341 + placeholder,
342 + processor,
343 + config: video_preprocessor_config,
344 + })
345 + }
346 + });
347 +
348 + if image.is_none() && video.is_none() {
349 + warn!(
350 + model_id = context.model_id,
351 + model_type = context.model_type,
352 + "no multimodal modality resolved; disabling multimodal support for this model"
353 + );
354 + return Ok(None);
355 + }
356 +
357 + let media_connector = Arc::new(MediaConnector::new(
358 + reqwest::Client::new(),
359 + MediaConnectorConfig::default(),
360 + )?);
361
362 Ok(Some(Self {
363 context,
216 - spec,
217 - image_processor: ResolvedImageProcessor {
218 - raw: image_processor,
219 - config: preprocessor_config,
220 - },
364 + spec: ResolvedMultimodalSpec::new(raw_spec),
365 + image,
366 + video,
367 media_connector,
368 }))
369 }
370
225 - /// Return the template-visible placeholder token for this model.
371 + /// Return the template-visible placeholder token for one modality, when
372 + /// this model supports it.
373 ///
227 - /// The HF renderer uses this token while flattening image content in string
228 - /// content format.
229 - pub fn placeholder_token(&self) -> &str {
230 - &self.spec.placeholder_token
374 + /// The HF renderer uses these tokens while flattening media content in
375 + /// string content format.
376 + pub fn placeholder_token(&self, modality: Modality) -> Option<&str> {
377 + match modality {
378 + Modality::Image => self.image.as_ref()?.placeholder.token.as_str().into(),
379 + Modality::Video => self.video.as_ref()?.placeholder.token.as_str().into(),
380 + _ => None,
381 + }
382 }
383 }
384
385 /// Finalize a rendered chat prompt into text-generation input.
386 ///
387 /// Text-only requests pass through unchanged as `Prompt::Text`. Multimodal
237 -/// requests are tokenized in chat, their image placeholders are expanded, and
238 -/// preprocessed image features are attached for engine-core transport.
388 +/// requests are tokenized in chat, their media placeholders are expanded, and
389 +/// preprocessed media features are attached for engine-core transport.
390 pub(crate) async fn finalize_rendered_prompt(
391 request: &ChatRequest,
392 rendered: RenderedPrompt,
@@ -260,7 +411,7 @@ pub(crate) async fn finalize_rendered_prompt(
411 Ok((Prompt::TokenIds(prompt_token_ids), Some(prepared)))
412 }
413
263 -/// Extract image media parts from chat messages in message/content order.
414 +/// Extract media parts from chat messages in message/content order.
415 ///
416 /// Assistant history is skipped because generated assistant blocks are already
417 /// represented as text for prompt rendering in this crate.
@@ -289,6 +440,12 @@ fn extract_media_parts(request: &ChatRequest) -> Result<Vec<MediaContentPart>> {
440 detail: *detail,
441 uuid: uuid.clone(),
442 }),
443 + ChatContentPart::VideoUrl { video_url, uuid } => {
444 + all_parts.push(MediaContentPart::VideoUrl {
445 + url: video_url.clone(),
446 + uuid: uuid.clone(),
447 + })
448 + }
449 }
450 }
451 }
@@ -296,8 +453,8 @@ fn extract_media_parts(request: &ChatRequest) -> Result<Vec<MediaContentPart>> {
453 }
454
455 impl MultimodalModelInfo {
299 - /// Run media fetch, image preprocessing, prompt expansion, and feature
300 - /// build.
456 + /// Run media fetch, per-modality preprocessing, prompt expansion, and
457 + /// feature build.
458 ///
459 /// `prompt_token_ids` is mutated in place because placeholder expansion
460 /// changes both the final prompt and the offsets recorded in
@@ -313,12 +470,47 @@ impl MultimodalModelInfo {
470 }
471 let media_parts_len = media_parts.len();
472
316 - let fetched = self.fetch_images(media_parts).await?;
317 - let preprocessed = self.preprocess_images(&fetched.frames).await?;
318 - let replacements = self.spec.prompt_replacements(&self.context, &preprocessed)?;
319 - let ranges = self.expand_prompt_tokens(prompt_token_ids, replacements)?;
473 + // TODO: enforce per-modality item-count limits, aligned with the
474 + // engine's `--limit-mm-per-prompt` semantics.
475 + let fetched = self.fetch_media(media_parts).await?;
476 +
477 + let mut prepared = Vec::new();
478 + if !fetched.images.is_empty() {
479 + prepared
480 + .push(self.prepare_images(fetched.images, fetched.image_uuids, model_dtype).await?);
481 + }
482 + if !fetched.videos.is_empty() {
483 + prepared
484 + .push(self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype).await?);
485 + }
486 +
487 + let mut ranges = expand_prompt_token_ids(prompt_token_ids, &prepared)?;
488 +
489 + let mut features = Vec::with_capacity(media_parts_len);
490 + for media in prepared {
491 + let media_ranges = ranges.remove(&media.modality).unwrap_or_default();
492 + if media_ranges.len() != media.items.len() {
493 + bail_multimodal!(
494 + "number of expanded `{}` placeholders {} does not match number of media items {}",
495 + media.modality,
496 + media_ranges.len(),
497 + media.items.len()
498 + );
499 + }
500 + for (item, range) in izip!(media.items, media_ranges) {
501 + features.push(MmFeatureSpec {
502 + data: Some(item.data),
503 + modality: media.modality.to_string(),
504 + identifier: item.uuid.unwrap_or_else(|| item.hash.clone()),
505 + mm_position: range,
506 + mm_hash: Some(item.hash),
507 + });
508 + }
509 + }
510 + // Mirror the Python frontend (`argsort_mm_positions`): features are
511 + // ordered by their placeholder position in the prompt.
512 + features.sort_by_key(|feature| feature.mm_position.offset);
513
321 - let features = self.build_features(preprocessed, fetched, ranges, model_dtype)?;
514 if features.len() != media_parts_len {
515 bail_multimodal!(
516 "number of built multimodal features {} does not match number of media parts {}",
@@ -329,219 +521,51 @@ impl MultimodalModelInfo {
521 Ok(features)
522 }
523
332 - /// Fetch all image parts and preserve their request-order UUID metadata.
333 - async fn fetch_images(&self, media_parts: Vec<MediaContentPart>) -> Result<FetchedImageMedia> {
524 + /// Fetch all media parts and split them per modality, preserving their
525 + /// request-order UUID metadata.
526 + async fn fetch_media(&self, media_parts: Vec<MediaContentPart>) -> Result<FetchedMedia> {
527 let mut tracker = AsyncMultiModalTracker::new(Arc::clone(&self.media_connector));
528 for part in media_parts {
336 - tracker.push_part(part).map_err(|error| multimodal!("{error}"))?;
529 + tracker.push_part(part)?;
530 }
531
339 - let tracker_output = tracker.finalize().await.map_err(|error| multimodal!("{error}"))?;
340 - let images = tracker_output.data.get(&Modality::Image).cloned().unwrap_or_default();
341 - let uuids = tracker_output.uuids.get(&Modality::Image).cloned().unwrap_or_default();
532 + let mut tracker_output = tracker.finalize().await?;
533
343 - let frames = images
534 + let images = tracker_output
535 + .data
536 + .remove(&Modality::Image)
537 + .unwrap_or_default()
538 .into_iter()
539 .map(|media| match media {
540 TrackedMedia::Image(frame) => Ok(frame),
347 - _ => Err(Error::UnsupportedMultimodalContent("non-image")),
541 + _ => Err(multimodal!(
542 + "tracker returned non-image media for the image modality"
543 + )),
544 })
545 .collect::<Result<Vec<_>>>()?;
546 + let image_uuids = tracker_output.uuids.remove(&Modality::Image).unwrap_or_default();
547
351 - Ok(FetchedImageMedia { frames, uuids })
352 - }
548 + let videos = tracker_output
549 + .data
550 + .remove(&Modality::Video)
551 + .unwrap_or_default()
552 + .into_iter()
553 + .map(|media| match media {
554 + TrackedMedia::Video(clip) => Ok(clip),
555 + _ => Err(multimodal!(
556 + "tracker returned non-video media for the video modality"
557 + )),
558 + })
559 + .collect::<Result<Vec<_>>>()?;
560 + let video_uuids = tracker_output.uuids.remove(&Modality::Video).unwrap_or_default();
561
354 - /// Preprocess fetched image frames with the model's resolved image
355 - /// processor.
356 - ///
357 - /// The processor work is CPU-heavy relative to request wiring, so it runs
358 - /// in a blocking task and returns owned tensors ready for wire
359 - /// conversion.
360 - async fn preprocess_images(
361 - &self,
362 - image_frames: &[Arc<llm_multimodal::ImageFrame>],
363 - ) -> Result<PreprocessedImages> {
364 - let config = self.image_processor.config.clone();
365 - let processor = self.image_processor.raw;
366 - let images = image_frames.iter().map(|frame| frame.data().clone()).collect::<Vec<_>>();
367 -
368 - // TODO: is it still necessary given that we've already in a dedicated runtime?
369 - tokio::task::spawn_blocking(move || {
370 - processor.preprocess(&images, &config).map_err(|error| multimodal!("{error}"))
562 + Ok(FetchedMedia {
563 + images,
564 + image_uuids,
565 + videos,
566 + video_uuids,
567 })
372 - .await
373 - .map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
374 - }
375 -
376 - /// Replace rendered placeholder markers with model-specific replacement
377 - /// tokens.
378 - ///
379 - /// Replacements are consumed in order, matching the original media-part
380 - /// order. The returned ranges point into the already-expanded prompt.
381 - fn expand_prompt_tokens(
382 - &self,
383 - prompt_token_ids: &mut Vec<u32>,
384 - replacements: Vec<PromptReplacement>,
385 - ) -> Result<Vec<PlaceholderRange>> {
386 - expand_prompt_token_ids(
387 - prompt_token_ids,
388 - replacements,
389 - self.spec.placeholder_marker_token_id,
390 - self.spec.placeholder_embed_token_id,
391 - &self.spec.placeholder_token,
392 - )
568 }
394 -
395 - /// Convert preprocessed image tensors into engine-core multimodal features.
396 - ///
397 - /// One `MmFeatureSpec` is produced per image. Tensor fields are
398 - /// sliced according to the model spec's field layout declarations.
399 - fn build_features(
400 - &self,
401 - preprocessed: PreprocessedImages,
402 - images: FetchedImageMedia,
403 - ranges: Vec<PlaceholderRange>,
404 - model_dtype: ModelDtype,
405 - ) -> Result<MmFeatures> {
406 - let len = images.frames.len();
407 - let tensors = tensor::collect_tensors(preprocessed, model_dtype)?;
408 -
409 - let mut features = Vec::with_capacity(images.frames.len());
410 - for (index, (frame, uuid, range)) in izip!(images.frames, images.uuids, ranges).enumerate()
411 - {
412 - let mut data = MmKwargsItem::new();
413 - for (key, tensor) in &tensors {
414 - let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(key);
415 - let (value, field) = match self.spec.field_layouts.get(key) {
416 - Some(FieldLayout::Batched) => (
417 - tensor.batched_value_at(index)?,
418 - MmField::Batched(MmBatchedField { keep_on_cpu }),
419 - ),
420 - Some(FieldLayout::Flat { sizes_key }) => {
421 - let sizes = tensors.get(sizes_key).ok_or_else(|| {
422 - multimodal!("flat tensor sizes key `{sizes_key}` is missing")
423 - })?;
424 - let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?;
425 - (
426 - tensor.flat_value_range(start, end)?,
427 - MmField::Flat(MmFlatField {
428 - slices: vec![MmSlice::Slice(SliceSpec {
429 - start: Some(0),
430 - stop: Some((end - start) as isize),
431 - step: None,
432 - })],
433 - dim: 0,
434 - keep_on_cpu,
435 - }),
436 - )
437 - }
438 - None => (
439 - tensor.clone(),
440 - MmField::Shared(MmSharedField {
441 - batch_size: len,
442 - keep_on_cpu,
443 - }),
444 - ),
445 - };
446 -
447 - data.insert(
448 - key.clone(),
449 - MmFieldElem {
450 - data: Some(value.try_into()?),
451 - field,
452 - },
453 - );
454 - }
455 -
456 - let hash = frame.hash.clone();
457 - features.push(MmFeatureSpec {
458 - data: Some(data),
459 - modality: "image".to_string(),
460 - identifier: uuid.unwrap_or_else(|| hash.clone()),
461 - mm_position: range,
462 - mm_hash: Some(hash),
463 - });
464 - }
465 -
466 - Ok(features)
467 - }
468 -}
469 -
470 -fn expand_prompt_token_ids(
471 - prompt_token_ids: &mut Vec<u32>,
472 - replacements: Vec<PromptReplacement>,
473 - placeholder_marker_token_id: u32,
474 - placeholder_embed_token_id: u32,
475 - placeholder_token: &str,
476 -) -> Result<Vec<PlaceholderRange>> {
477 - if replacements.is_empty() {
478 - return Ok(Vec::new());
479 - }
480 -
481 - let replacement_growth = replacements.iter().fold(0usize, |total, replacement| {
482 - total.saturating_add(replacement.tokens.len().saturating_sub(1))
483 - });
484 - let mut expanded =
485 - Vec::with_capacity(prompt_token_ids.len().saturating_add(replacement_growth));
486 - let mut ranges = Vec::with_capacity(replacements.len());
487 - let mut cursor = 0usize;
488 -
489 - for replacement in replacements {
490 - if replacement.modality != Modality::Image {
491 - bail_multimodal!(
492 - "unsupported prompt replacement modality `{}`",
493 - replacement.modality
494 - );
495 - }
496 -
497 - let offset = find_next_token(prompt_token_ids, placeholder_marker_token_id, cursor)
498 - .ok_or_else(|| {
499 - multimodal!(
500 - "placeholder token `{placeholder_token}` was not found in tokenized prompt"
501 - )
502 - })?;
503 -
504 - if replacement.tokens.is_empty() {
505 - bail_multimodal!("placeholder token `{placeholder_token}` expanded to no tokens");
506 - }
507 -
508 - let replacement_len = replacement.tokens.len();
509 - let is_embed = {
510 - let mask = replacement
511 - .tokens
512 - .iter()
513 - .map(|&token| token as u32 == placeholder_embed_token_id)
514 - .collect::<Vec<_>>();
515 - WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)?
516 - };
517 -
518 - expanded.extend_from_slice(&prompt_token_ids[cursor..offset]);
519 - let expanded_offset = expanded.len();
520 - expanded.extend(replacement.tokens.into_iter().map(|token| token as u32));
521 - ranges.push(PlaceholderRange {
522 - offset: expanded_offset,
523 - length: replacement_len,
524 - is_embed: Some(is_embed),
525 - });
526 - cursor = offset + 1;
527 - }
528 -
529 - expanded.extend_from_slice(&prompt_token_ids[cursor..]);
530 - *prompt_token_ids = expanded;
531 -
532 - Ok(ranges)
533 -}
534 -
535 -/// Find `needle` in `haystack`, starting at `start`.
536 -///
537 -/// This is intentionally order-preserving rather than a global replace: each
538 -/// image consumes the next placeholder occurrence.
539 -fn find_next_token(haystack: &[u32], needle: u32, start: usize) -> Option<usize> {
540 - haystack
541 - .get(start..)?
542 - .iter()
543 - .position(|token| *token == needle)
544 - .map(|offset| start + offset)
569 }
570
571 /// Adapter from the frontend tokenizer trait to `llm-multimodal`.
@@ -566,18 +590,19 @@ impl TokenResolver for TokenizerResolver {
590 mod tests {
591 use std::sync::Arc;
592
569 - use llm_multimodal::TokenId;
570 - use vllm_engine_core_client::protocol::tensor::WireArrayData;
593 use vllm_tokenizer::test_utils::TestTokenizer;
594
595 use super::*;
596
575 - const LLAMA4_IMAGE_START_ID: u32 = 200088;
576 - const LLAMA4_IMAGE_END_ID: u32 = 200089;
577 - const LLAMA4_IMAGE_ID: u32 = 200090;
578 - const LLAMA4_PATCH_ID: u32 = 200092;
579 - const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093;
580 - const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094;
597 + pub(super) const LLAMA4_IMAGE_START_ID: u32 = 200088;
598 + pub(super) const LLAMA4_IMAGE_END_ID: u32 = 200089;
599 + pub(super) const LLAMA4_IMAGE_ID: u32 = 200090;
600 + pub(super) const LLAMA4_PATCH_ID: u32 = 200092;
601 + pub(super) const LLAMA4_TILE_X_SEPARATOR_ID: u32 = 200093;
602 + pub(super) const LLAMA4_TILE_Y_SEPARATOR_ID: u32 = 200094;
603 +
604 + pub(super) const QWEN3_IMAGE_PAD_ID: u32 = 151655;
605 + pub(super) const QWEN3_VIDEO_PAD_ID: u32 = 151656;
606
607 fn llama4_tokenizer() -> TestTokenizer {
608 TestTokenizer::new()
@@ -589,33 +614,31 @@ mod tests {
614 .with_regular_token("<|tile_y_separator|>", LLAMA4_TILE_Y_SEPARATOR_ID)
615 }
616
592 - fn test_info(model_type: &str, config: serde_json::Value) -> MultimodalModelInfo {
617 + pub(super) fn qwen3_vl_tokenizer() -> TestTokenizer {
618 + TestTokenizer::new()
619 + .with_regular_token("<|image_pad|>", QWEN3_IMAGE_PAD_ID)
620 + .with_regular_token("<|video_pad|>", QWEN3_VIDEO_PAD_ID)
621 + }
622 +
623 + fn test_info(
624 + model_type: &str,
625 + config: serde_json::Value,
626 + tokenizer: TestTokenizer,
627 + ) -> MultimodalModelInfo {
628 let context = MultimodalModelContext {
629 model_id: format!("{model_type}-test"),
630 model_type: Some(model_type.to_string()),
631 config,
597 - tokenizer: TokenizerResolver(Arc::new(llama4_tokenizer())),
632 + tokenizer: TokenizerResolver(Arc::new(tokenizer)),
633 };
599 - let spec = context
600 - .resolve_model_spec()
601 - .unwrap_or_else(|| panic!("{model_type} spec should match"));
602 - let spec = ResolvedMultimodalSpec::new(spec, &context).unwrap();
603 - let raw_image_processor = context
604 - .resolve_image_processor()
605 - .unwrap_or_else(|| panic!("{model_type} image processor should match"));
606 - let media_connector = Arc::new(
607 - MediaConnector::new(reqwest::Client::new(), MediaConnectorConfig::default()).unwrap(),
608 - );
634
610 - MultimodalModelInfo {
635 + MultimodalModelInfo::from_loaded(
636 context,
612 - spec,
613 - image_processor: ResolvedImageProcessor {
614 - raw: raw_image_processor,
615 - config: PreProcessorConfig::default(),
616 - },
617 - media_connector,
618 - }
637 + PreProcessorConfig::default(),
638 + PreProcessorConfig::default(),
639 + )
640 + .unwrap()
641 + .unwrap_or_else(|| panic!("{model_type} multimodal support should resolve"))
642 }
643
644 fn llama4_info() -> MultimodalModelInfo {
@@ -624,173 +647,96 @@ mod tests {
647 "image_token_index": LLAMA4_PATCH_ID,
648 "vision_config": {"image_size": 336, "patch_size": 14}
649 });
627 - test_info("llama4", config)
650 + test_info("llama4", config, llama4_tokenizer())
651 }
652
630 - fn llama4_single_tile_replacement() -> PromptReplacement {
631 - PromptReplacement::sequence(
632 - Modality::Image,
633 - "<|image|>",
634 - vec![
635 - LLAMA4_IMAGE_START_ID as TokenId,
636 - LLAMA4_IMAGE_ID as TokenId,
637 - LLAMA4_PATCH_ID as TokenId,
638 - LLAMA4_PATCH_ID as TokenId,
639 - LLAMA4_IMAGE_END_ID as TokenId,
640 - ],
641 - )
653 + pub(super) fn qwen3_vl_info() -> MultimodalModelInfo {
654 + let config = serde_json::json!({
655 + "model_type": "qwen3_vl",
656 + "image_token_id": QWEN3_IMAGE_PAD_ID,
657 + "video_token_id": QWEN3_VIDEO_PAD_ID,
658 + "vision_start_token_id": 151652,
659 + "vision_end_token_id": 151653,
660 + "vision_config": {"patch_size": 16}
661 + });
662 + test_info("qwen3_vl", config, qwen3_vl_tokenizer())
663 }
664
644 - fn llama4_multi_tile_replacement() -> PromptReplacement {
645 - PromptReplacement::sequence(
646 - Modality::Image,
647 - "<|image|>",
648 - vec![
649 - LLAMA4_IMAGE_START_ID as TokenId,
650 - LLAMA4_PATCH_ID as TokenId,
651 - LLAMA4_TILE_X_SEPARATOR_ID as TokenId,
652 - LLAMA4_PATCH_ID as TokenId,
653 - LLAMA4_TILE_Y_SEPARATOR_ID as TokenId,
654 - LLAMA4_IMAGE_ID as TokenId,
655 - LLAMA4_PATCH_ID as TokenId,
656 - LLAMA4_IMAGE_END_ID as TokenId,
657 - ],
665 + #[test]
666 + fn from_paths_resolves_image_config_from_processor_config() {
667 + let dir = tempfile::tempdir().unwrap();
668 + let config_path = dir.path().join("config.json");
669 + std::fs::write(
670 + &config_path,
671 + serde_json::json!({
672 + "model_type": "qwen3_vl",
673 + "image_token_id": QWEN3_IMAGE_PAD_ID,
674 + })
675 + .to_string(),
676 )
659 - }
677 + .unwrap();
678 + let processor_config_path = dir.path().join("processor_config.json");
679 + std::fs::write(
680 + &processor_config_path,
681 + r#"{"image_processor":{"size":{"shortest_edge":64}}}"#,
682 + )
683 + .unwrap();
684 +
685 + let info = MultimodalModelInfo::from_paths(
686 + "qwen3-vl-test".to_string(),
687 + Some("qwen3_vl".to_string()),
688 + MultimodalConfigFiles {
689 + config: Some(&config_path),
690 + processor_config: Some(&processor_config_path),
691 + ..Default::default()
692 + },
693 + Arc::new(qwen3_vl_tokenizer()),
694 + )
695 + .unwrap()
696 + .unwrap();
697
661 - fn assert_bool_mask(range: &PlaceholderRange, expected: &[bool]) {
662 - let tensor = range.is_embed.as_ref().expect("is_embed mask");
663 - assert_eq!(tensor.dtype, "bool");
664 - assert_eq!(tensor.shape, vec![expected.len()]);
665 - assert_eq!(
666 - tensor.data,
667 - WireArrayData::RawView(expected.iter().map(|value| u8::from(*value)).collect())
668 - );
698 + assert_eq!(info.image.unwrap().config.get_shortest_edge(), Some(64));
699 }
700
701 #[test]
672 - fn expand_prompt_tokens_marks_only_llama4_patch_tokens_as_embed() {
673 - let info = llama4_info();
674 - let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
675 - let replacements = vec![llama4_multi_tile_replacement()];
676 -
677 - let ranges = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap();
702 + fn qwen3_vl_resolves_image_and_video_support() {
703 + let info = qwen3_vl_info();
704
705 assert_eq!(
680 - prompt_token_ids,
681 - vec![
682 - 1,
683 - LLAMA4_IMAGE_START_ID,
684 - LLAMA4_PATCH_ID,
685 - LLAMA4_TILE_X_SEPARATOR_ID,
686 - LLAMA4_PATCH_ID,
687 - LLAMA4_TILE_Y_SEPARATOR_ID,
688 - LLAMA4_IMAGE_ID,
689 - LLAMA4_PATCH_ID,
690 - LLAMA4_IMAGE_END_ID,
691 - 2,
692 - ]
706 + info.placeholder_token(Modality::Image),
707 + Some("<|image_pad|>")
708 );
694 - assert_eq!(ranges[0].offset, 1);
695 - assert_eq!(ranges[0].length, 8);
696 - assert_bool_mask(
697 - &ranges[0],
698 - &[false, true, false, true, false, false, true, false],
709 + assert_eq!(
710 + info.placeholder_token(Modality::Video),
711 + Some("<|video_pad|>")
712 + );
713 + assert_ne!(
714 + info.image.as_ref().unwrap().placeholder.marker_token_id,
715 + info.video.as_ref().unwrap().placeholder.marker_token_id,
716 );
717 }
718
719 #[test]
703 - fn expand_prompt_tokens_errors_when_placeholder_missing() {
704 - let info = llama4_info();
705 - let mut prompt_token_ids = vec![1, 2, 3];
706 - let replacements = vec![llama4_single_tile_replacement()];
707 -
708 - let error = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap_err();
709 -
710 - assert!(matches!(error, Error::Multimodal(message) if message.contains("not found")));
711 - }
712 -
713 - #[test]
714 - fn expand_prompt_tokens_ignores_empty_replacements() {
715 - let info = llama4_info();
716 - let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
717 - let original_prompt_token_ids = prompt_token_ids.clone();
718 -
719 - let ranges = info.expand_prompt_tokens(&mut prompt_token_ids, Vec::new()).unwrap();
720 -
721 - assert!(ranges.is_empty());
722 - assert_eq!(prompt_token_ids, original_prompt_token_ids);
723 - }
724 -
725 - #[test]
726 - fn expand_prompt_tokens_leaves_prompt_unchanged_when_later_placeholder_missing() {
727 - let info = llama4_info();
728 - let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
729 - let original_prompt_token_ids = prompt_token_ids.clone();
730 - let replacements = vec![
731 - llama4_single_tile_replacement(),
732 - llama4_single_tile_replacement(),
733 - ];
734 -
735 - let error = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap_err();
736 -
737 - assert!(matches!(error, Error::Multimodal(message) if message.contains("not found")));
738 - assert_eq!(prompt_token_ids, original_prompt_token_ids);
739 - }
720 + fn qwen3_vl_without_video_token_id_disables_video_support_only() {
721 + let config = serde_json::json!({
722 + "model_type": "qwen3_vl",
723 + "image_token_id": QWEN3_IMAGE_PAD_ID,
724 + "vision_config": {"patch_size": 16}
725 + });
726 + let info = test_info("qwen3_vl", config, qwen3_vl_tokenizer());
727
741 - #[test]
742 - fn expand_prompt_tokens_errors_when_replacement_is_empty() {
743 - let info = llama4_info();
744 - let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
745 - let original_prompt_token_ids = prompt_token_ids.clone();
746 - let replacements = vec![PromptReplacement::sequence(
747 - Modality::Image,
748 - "<|image|>",
749 - Vec::new(),
750 - )];
751 -
752 - let error = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap_err();
753 -
754 - assert!(
755 - matches!(error, Error::Multimodal(message) if message.contains("expanded to no tokens"))
728 + assert_eq!(
729 + info.placeholder_token(Modality::Image),
730 + Some("<|image_pad|>")
731 );
757 - assert_eq!(prompt_token_ids, original_prompt_token_ids);
732 + assert_eq!(info.placeholder_token(Modality::Video), None);
733 }
734
735 #[test]
761 - fn expand_prompt_tokens_skips_llama4_image_marker_inside_replacement() {
736 + fn llama4_resolves_image_support_only() {
737 let info = llama4_info();
763 - let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2, LLAMA4_IMAGE_ID, 3];
764 - let replacements = vec![
765 - llama4_single_tile_replacement(),
766 - llama4_single_tile_replacement(),
767 - ];
768 -
769 - let ranges = info.expand_prompt_tokens(&mut prompt_token_ids, replacements).unwrap();
738
771 - assert_eq!(
772 - prompt_token_ids,
773 - vec![
774 - 1,
775 - LLAMA4_IMAGE_START_ID,
776 - LLAMA4_IMAGE_ID,
777 - LLAMA4_PATCH_ID,
778 - LLAMA4_PATCH_ID,
779 - LLAMA4_IMAGE_END_ID,
780 - 2,
781 - LLAMA4_IMAGE_START_ID,
782 - LLAMA4_IMAGE_ID,
783 - LLAMA4_PATCH_ID,
784 - LLAMA4_PATCH_ID,
785 - LLAMA4_IMAGE_END_ID,
786 - 3,
787 - ]
788 - );
789 - assert_eq!(ranges[0].offset, 1);
790 - assert_eq!(ranges[0].length, 5);
791 - assert_bool_mask(&ranges[0], &[false, false, true, true, false]);
792 - assert_eq!(ranges[1].offset, 7);
793 - assert_eq!(ranges[1].length, 5);
794 - assert_bool_mask(&ranges[1], &[false, false, true, true, false]);
739 + assert_eq!(info.placeholder_token(Modality::Image), Some("<|image|>"));
740 + assert_eq!(info.placeholder_token(Modality::Video), None);
741 }
742 }
rust/src/chat/src/multimodal/expand.rs new
+446
@@ -0,0 +1,446 @@
1 +//! Prompt placeholder expansion shared across modalities.
2 +
3 +use std::collections::{HashMap, VecDeque};
4 +
5 +use llm_multimodal::{Modality, PromptReplacement};
6 +use vllm_engine_core_client::protocol::multimodal::PlaceholderRange;
7 +use vllm_engine_core_client::protocol::tensor::WireTensor;
8 +
9 +use super::PreparedMedia;
10 +use crate::error::{Error, Result, bail_multimodal};
11 +
12 +/// One modality's queue of pending placeholder replacements for prompt
13 +/// expansion.
14 +struct ExpansionLane<'a> {
15 + modality: Modality,
16 + marker_token_id: u32,
17 + embed_token_id: u32,
18 + placeholder_token: String,
19 + replacements: VecDeque<&'a PromptReplacement>,
20 +}
21 +
22 +impl<'a> ExpansionLane<'a> {
23 + fn from_prepared(media: &'a PreparedMedia) -> Option<Self> {
24 + if media.replacements.is_empty() {
25 + return None;
26 + }
27 +
28 + Some(Self {
29 + modality: media.modality,
30 + marker_token_id: media.placeholder.marker_token_id,
31 + embed_token_id: media.placeholder.embed_token_id,
32 + placeholder_token: media.placeholder.token.clone(),
33 + replacements: media.replacements.iter().collect(),
34 + })
35 + }
36 +}
37 +
38 +/// Replace rendered placeholder markers with model-specific replacement
39 +/// tokens across all modalities in one left-to-right pass.
40 +///
41 +/// Each prepared modality consumes its own marker occurrences in order,
42 +/// matching the original media-part order within that modality; markers of
43 +/// different modalities may interleave freely.
44 +///
45 +/// The returned ranges point into the already-expanded prompt, grouped per
46 +/// modality in item order.
47 +pub(super) fn expand_prompt_token_ids(
48 + prompt_token_ids: &mut Vec<u32>,
49 + prepared: &[PreparedMedia],
50 +) -> Result<HashMap<Modality, Vec<PlaceholderRange>>> {
51 + let mut lanes = prepared.iter().filter_map(ExpansionLane::from_prepared).collect::<Vec<_>>();
52 + if lanes.is_empty() {
53 + return Ok(HashMap::new());
54 + }
55 +
56 + let replacement_growth = lanes
57 + .iter()
58 + .flat_map(|lane| lane.replacements.iter())
59 + .fold(0usize, |total, replacement| {
60 + total.saturating_add(replacement.tokens.len().saturating_sub(1))
61 + });
62 + let expanded_len = prompt_token_ids.len().saturating_add(replacement_growth);
63 +
64 + let mut expanded = Vec::with_capacity(expanded_len);
65 + let mut ranges = HashMap::<Modality, Vec<PlaceholderRange>>::new();
66 +
67 + for &token in prompt_token_ids.iter() {
68 + let lane = lanes
69 + .iter_mut()
70 + .find(|lane| lane.marker_token_id == token && !lane.replacements.is_empty());
71 + let Some(lane) = lane else {
72 + expanded.push(token);
73 + continue;
74 + };
75 +
76 + let replacement = lane.replacements.pop_front().expect("lane queue is non-empty");
77 + debug_assert_eq!(replacement.modality, lane.modality);
78 + if replacement.tokens.is_empty() {
79 + bail_multimodal!(
80 + "placeholder token `{}` expanded to no tokens",
81 + lane.placeholder_token
82 + );
83 + }
84 +
85 + let replacement_len = replacement.tokens.len();
86 + let is_embed = {
87 + let mask = replacement
88 + .tokens
89 + .iter()
90 + .map(|&token| token as u32 == lane.embed_token_id)
91 + .collect::<Vec<_>>();
92 + WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)?
93 + };
94 +
95 + let expanded_offset = expanded.len();
96 + expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
97 + ranges.entry(lane.modality).or_default().push(PlaceholderRange {
98 + offset: expanded_offset,
99 + length: replacement_len,
100 + is_embed: Some(is_embed),
101 + });
102 + }
103 +
104 + for lane in &lanes {
105 + if !lane.replacements.is_empty() {
106 + bail_multimodal!(
107 + "placeholder token `{}` was not found in tokenized prompt for {} remaining `{}` item(s)",
108 + lane.placeholder_token,
109 + lane.replacements.len(),
110 + lane.modality
111 + );
112 + }
113 + }
114 +
115 + *prompt_token_ids = expanded;
116 +
117 + Ok(ranges)
118 +}
119 +
120 +#[cfg(test)]
121 +mod tests {
122 + use llm_multimodal::TokenId;
123 + use vllm_engine_core_client::protocol::tensor::WireArrayData;
124 +
125 + use super::super::tests::{
126 + LLAMA4_IMAGE_END_ID, LLAMA4_IMAGE_ID, LLAMA4_IMAGE_START_ID, LLAMA4_PATCH_ID,
127 + LLAMA4_TILE_X_SEPARATOR_ID, LLAMA4_TILE_Y_SEPARATOR_ID, QWEN3_IMAGE_PAD_ID,
128 + QWEN3_VIDEO_PAD_ID,
129 + };
130 + use super::super::{PreparedMedia, ResolvedPlaceholder};
131 + use super::*;
132 +
133 + /// Build prepared media directly from placeholder token IDs.
134 + fn prepared_media(
135 + modality: Modality,
136 + placeholder_token: &str,
137 + marker_token_id: u32,
138 + embed_token_id: u32,
139 + replacements: Vec<PromptReplacement>,
140 + ) -> PreparedMedia {
141 + PreparedMedia {
142 + modality,
143 + placeholder: ResolvedPlaceholder {
144 + token: placeholder_token.to_string(),
145 + marker_token_id,
146 + embed_token_id,
147 + },
148 + replacements,
149 + items: Vec::new(),
150 + }
151 + }
152 +
153 + /// Llama4 image prepared media: the `<|image|>` marker expands to
154 + /// sequences whose embed positions are the `<|patch|>` tokens.
155 + fn llama4_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
156 + prepared_media(
157 + Modality::Image,
158 + "<|image|>",
159 + LLAMA4_IMAGE_ID,
160 + LLAMA4_PATCH_ID,
161 + replacements,
162 + )
163 + }
164 +
165 + fn qwen3_image_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
166 + prepared_media(
167 + Modality::Image,
168 + "<|image_pad|>",
169 + QWEN3_IMAGE_PAD_ID,
170 + QWEN3_IMAGE_PAD_ID,
171 + replacements,
172 + )
173 + }
174 +
175 + fn qwen3_video_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
176 + prepared_media(
177 + Modality::Video,
178 + "<|video_pad|>",
179 + QWEN3_VIDEO_PAD_ID,
180 + QWEN3_VIDEO_PAD_ID,
181 + replacements,
182 + )
183 + }
184 +
185 + fn llama4_single_tile_replacement() -> PromptReplacement {
186 + PromptReplacement::sequence(
187 + Modality::Image,
188 + "<|image|>",
189 + vec![
190 + LLAMA4_IMAGE_START_ID as TokenId,
191 + LLAMA4_IMAGE_ID as TokenId,
192 + LLAMA4_PATCH_ID as TokenId,
193 + LLAMA4_PATCH_ID as TokenId,
194 + LLAMA4_IMAGE_END_ID as TokenId,
195 + ],
196 + )
197 + }
198 +
199 + fn llama4_multi_tile_replacement() -> PromptReplacement {
200 + PromptReplacement::sequence(
201 + Modality::Image,
202 + "<|image|>",
203 + vec![
204 + LLAMA4_IMAGE_START_ID as TokenId,
205 + LLAMA4_PATCH_ID as TokenId,
206 + LLAMA4_TILE_X_SEPARATOR_ID as TokenId,
207 + LLAMA4_PATCH_ID as TokenId,
208 + LLAMA4_TILE_Y_SEPARATOR_ID as TokenId,
209 + LLAMA4_IMAGE_ID as TokenId,
210 + LLAMA4_PATCH_ID as TokenId,
211 + LLAMA4_IMAGE_END_ID as TokenId,
212 + ],
213 + )
214 + }
215 +
216 + fn assert_bool_mask(range: &PlaceholderRange, expected: &[bool]) {
217 + let tensor = range.is_embed.as_ref().expect("is_embed mask");
218 + assert_eq!(tensor.dtype, "bool");
219 + assert_eq!(tensor.shape, vec![expected.len()]);
220 + assert_eq!(
221 + tensor.data,
222 + WireArrayData::RawView(expected.iter().map(|value| u8::from(*value)).collect())
223 + );
224 + }
225 +
226 + #[test]
227 + fn expand_prompt_tokens_marks_only_llama4_patch_tokens_as_embed() {
228 + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
229 + let prepared = vec![llama4_prepared(vec![llama4_multi_tile_replacement()])];
230 +
231 + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
232 + let ranges = &ranges[&Modality::Image];
233 +
234 + assert_eq!(
235 + prompt_token_ids,
236 + vec![
237 + 1,
238 + LLAMA4_IMAGE_START_ID,
239 + LLAMA4_PATCH_ID,
240 + LLAMA4_TILE_X_SEPARATOR_ID,
241 + LLAMA4_PATCH_ID,
242 + LLAMA4_TILE_Y_SEPARATOR_ID,
243 + LLAMA4_IMAGE_ID,
244 + LLAMA4_PATCH_ID,
245 + LLAMA4_IMAGE_END_ID,
246 + 2,
247 + ]
248 + );
249 + assert_eq!(ranges[0].offset, 1);
250 + assert_eq!(ranges[0].length, 8);
251 + assert_bool_mask(
252 + &ranges[0],
253 + &[false, true, false, true, false, false, true, false],
254 + );
255 + }
256 +
257 + #[test]
258 + fn expand_prompt_tokens_errors_when_placeholder_missing() {
259 + let mut prompt_token_ids = vec![1, 2, 3];
260 + let prepared = vec![llama4_prepared(vec![llama4_single_tile_replacement()])];
261 +
262 + let error = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap_err();
263 +
264 + assert!(matches!(error, Error::Multimodal(message) if message.contains("not found")));
265 + }
266 +
267 + #[test]
268 + fn expand_prompt_tokens_ignores_empty_replacements() {
269 + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
270 + let original_prompt_token_ids = prompt_token_ids.clone();
271 + let prepared = vec![llama4_prepared(Vec::new())];
272 +
273 + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
274 +
275 + assert!(ranges.is_empty());
276 + assert_eq!(prompt_token_ids, original_prompt_token_ids);
277 + }
278 +
279 + #[test]
280 + fn expand_prompt_tokens_leaves_prompt_unchanged_when_later_placeholder_missing() {
281 + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
282 + let original_prompt_token_ids = prompt_token_ids.clone();
283 + let prepared = vec![llama4_prepared(vec![
284 + llama4_single_tile_replacement(),
285 + llama4_single_tile_replacement(),
286 + ])];
287 +
288 + let error = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap_err();
289 +
290 + assert!(matches!(error, Error::Multimodal(message) if message.contains("not found")));
291 + assert_eq!(prompt_token_ids, original_prompt_token_ids);
292 + }
293 +
294 + #[test]
295 + fn expand_prompt_tokens_errors_when_replacement_is_empty() {
296 + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2];
297 + let original_prompt_token_ids = prompt_token_ids.clone();
298 + let prepared = vec![llama4_prepared(vec![PromptReplacement::sequence(
299 + Modality::Image,
300 + "<|image|>",
301 + Vec::new(),
302 + )])];
303 +
304 + let error = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap_err();
305 +
306 + assert!(
307 + matches!(error, Error::Multimodal(message) if message.contains("expanded to no tokens"))
308 + );
309 + assert_eq!(prompt_token_ids, original_prompt_token_ids);
310 + }
311 +
312 + #[test]
313 + fn expand_prompt_tokens_skips_llama4_image_marker_inside_replacement() {
314 + let mut prompt_token_ids = vec![1, LLAMA4_IMAGE_ID, 2, LLAMA4_IMAGE_ID, 3];
315 + let prepared = vec![llama4_prepared(vec![
316 + llama4_single_tile_replacement(),
317 + llama4_single_tile_replacement(),
318 + ])];
319 +
320 + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
321 + let ranges = &ranges[&Modality::Image];
322 +
323 + assert_eq!(
324 + prompt_token_ids,
325 + vec![
326 + 1,
327 + LLAMA4_IMAGE_START_ID,
328 + LLAMA4_IMAGE_ID,
329 + LLAMA4_PATCH_ID,
330 + LLAMA4_PATCH_ID,
331 + LLAMA4_IMAGE_END_ID,
332 + 2,
333 + LLAMA4_IMAGE_START_ID,
334 + LLAMA4_IMAGE_ID,
335 + LLAMA4_PATCH_ID,
336 + LLAMA4_PATCH_ID,
337 + LLAMA4_IMAGE_END_ID,
338 + 3,
339 + ]
340 + );
341 + assert_eq!(ranges[0].offset, 1);
342 + assert_eq!(ranges[0].length, 5);
343 + assert_bool_mask(&ranges[0], &[false, false, true, true, false]);
344 + assert_eq!(ranges[1].offset, 7);
345 + assert_eq!(ranges[1].length, 5);
346 + assert_bool_mask(&ranges[1], &[false, false, true, true, false]);
347 + }
348 +
349 + #[test]
350 + fn expand_prompt_tokens_interleaves_image_and_video_prepared_media() {
351 + let mut prompt_token_ids = vec![
352 + 1,
353 + QWEN3_IMAGE_PAD_ID,
354 + 2,
355 + QWEN3_VIDEO_PAD_ID,
356 + 3,
357 + QWEN3_IMAGE_PAD_ID,
358 + 4,
359 + ];
360 + let prepared = vec![
361 + qwen3_image_prepared(vec![
362 + PromptReplacement::repeated(
363 + Modality::Image,
364 + "<|image_pad|>",
365 + QWEN3_IMAGE_PAD_ID as TokenId,
366 + 2,
367 + ),
368 + PromptReplacement::repeated(
369 + Modality::Image,
370 + "<|image_pad|>",
371 + QWEN3_IMAGE_PAD_ID as TokenId,
372 + 3,
373 + ),
374 + ]),
375 + qwen3_video_prepared(vec![PromptReplacement::repeated(
376 + Modality::Video,
377 + "<|video_pad|>",
378 + QWEN3_VIDEO_PAD_ID as TokenId,
379 + 4,
380 + )]),
381 + ];
382 +
383 + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
384 +
385 + assert_eq!(
386 + prompt_token_ids,
387 + vec![
388 + 1,
389 + QWEN3_IMAGE_PAD_ID,
390 + QWEN3_IMAGE_PAD_ID,
391 + 2,
392 + QWEN3_VIDEO_PAD_ID,
393 + QWEN3_VIDEO_PAD_ID,
394 + QWEN3_VIDEO_PAD_ID,
395 + QWEN3_VIDEO_PAD_ID,
396 + 3,
397 + QWEN3_IMAGE_PAD_ID,
398 + QWEN3_IMAGE_PAD_ID,
399 + QWEN3_IMAGE_PAD_ID,
400 + 4,
401 + ]
402 + );
403 +
404 + let image_ranges = &ranges[&Modality::Image];
405 + assert_eq!(image_ranges[0].offset, 1);
406 + assert_eq!(image_ranges[0].length, 2);
407 + assert_bool_mask(&image_ranges[0], &[true, true]);
408 + assert_eq!(image_ranges[1].offset, 9);
409 + assert_eq!(image_ranges[1].length, 3);
410 + assert_bool_mask(&image_ranges[1], &[true, true, true]);
411 +
412 + let video_ranges = &ranges[&Modality::Video];
413 + assert_eq!(video_ranges[0].offset, 4);
414 + assert_eq!(video_ranges[0].length, 4);
415 + assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
416 + }
417 +
418 + #[test]
419 + fn expand_prompt_tokens_error_names_modality_with_leftover_replacements() {
420 + let mut prompt_token_ids = vec![1, QWEN3_IMAGE_PAD_ID, 2];
421 + let original_prompt_token_ids = prompt_token_ids.clone();
422 + let prepared = vec![
423 + qwen3_image_prepared(vec![PromptReplacement::repeated(
424 + Modality::Image,
425 + "<|image_pad|>",
426 + QWEN3_IMAGE_PAD_ID as TokenId,
427 + 2,
428 + )]),
429 + qwen3_video_prepared(vec![PromptReplacement::repeated(
430 + Modality::Video,
431 + "<|video_pad|>",
432 + QWEN3_VIDEO_PAD_ID as TokenId,
433 + 4,
434 + )]),
435 + ];
436 +
437 + let error = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap_err();
438 +
439 + assert!(matches!(
440 + error,
441 + Error::Multimodal(message)
442 + if message.contains("<|video_pad|>") && message.contains("`video`")
443 + ));
444 + assert_eq!(prompt_token_ids, original_prompt_token_ids);
445 + }
446 +}
rust/src/chat/src/multimodal/image.rs new
+141
@@ -0,0 +1,141 @@
1 +//! Image-modality preparation: batch preprocessing and per-item feature
2 +//! build.
3 +
4 +use std::sync::Arc;
5 +
6 +use itertools::izip;
7 +use llm_multimodal::{FieldLayout, ImageFrame, Modality, PreprocessedEncoderInputs};
8 +use vllm_engine_core_client::protocol::dtype::ModelDtype;
9 +use vllm_engine_core_client::protocol::multimodal::{
10 + MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice,
11 + SliceSpec,
12 +};
13 +
14 +use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
15 +use crate::error::{Error, Result, bail_multimodal, multimodal};
16 +
17 +impl MultimodalModelInfo {
18 + /// Preprocess all fetched image frames as one batch and build per-item
19 + /// features.
20 + pub(super) async fn prepare_images(
21 + &self,
22 + frames: Vec<Arc<ImageFrame>>,
23 + uuids: Vec<Option<String>>,
24 + model_dtype: ModelDtype,
25 + ) -> Result<PreparedMedia> {
26 + let support = self.image.as_ref().ok_or_else(|| Error::UnsupportedModality {
27 + modality: Modality::Image.to_string(),
28 + })?;
29 + let preprocessed = self.preprocess_images(support, &frames).await?;
30 + let replacements =
31 + self.spec
32 + .prompt_replacements_for(&self.context, &preprocessed, Modality::Image)?;
33 + if replacements.len() != frames.len() {
34 + bail_multimodal!(
35 + "number of image prompt replacements {} does not match number of images {}",
36 + replacements.len(),
37 + frames.len()
38 + );
39 + }
40 + let items = self.build_image_items(preprocessed, &frames, uuids, model_dtype)?;
41 +
42 + Ok(PreparedMedia {
43 + modality: Modality::Image,
44 + placeholder: support.placeholder.clone(),
45 + replacements,
46 + items,
47 + })
48 + }
49 +
50 + /// Preprocess fetched image frames with the model's resolved vision
51 + /// processor.
52 + ///
53 + /// The processor work is CPU-heavy relative to request wiring, so it runs
54 + /// in a blocking task and returns owned tensors ready for wire
55 + /// conversion.
56 + async fn preprocess_images(
57 + &self,
58 + support: &ModalitySupport,
59 + image_frames: &[Arc<ImageFrame>],
60 + ) -> Result<PreprocessedEncoderInputs> {
61 + let config = support.config.clone();
62 + let processor = support.processor;
63 + let images = image_frames.iter().map(|frame| frame.data().clone()).collect::<Vec<_>>();
64 +
65 + // TODO: is it still necessary given that we've already in a dedicated runtime?
66 + tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images, &config)?))
67 + .await
68 + .map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
69 + }
70 +
71 + /// Convert one batch of preprocessed image tensors into per-item engine
72 + /// kwargs.
73 + ///
74 + /// Tensor fields are sliced per item according to the model spec's field
75 + /// layout declarations.
76 + fn build_image_items(
77 + &self,
78 + preprocessed: PreprocessedEncoderInputs,
79 + frames: &[Arc<ImageFrame>],
80 + uuids: Vec<Option<String>>,
81 + model_dtype: ModelDtype,
82 + ) -> Result<Vec<PreparedItem>> {
83 + let len = frames.len();
84 + let tensors = tensor::collect_tensors(preprocessed, "pixel_values", model_dtype)?;
85 +
86 + let mut items = Vec::with_capacity(len);
87 + for (index, (frame, uuid)) in izip!(frames, uuids).enumerate() {
88 + let mut data = MmKwargsItem::new();
89 + for (key, tensor) in &tensors {
90 + let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(key);
91 + let (value, field) = match self.spec.field_layouts.get(key) {
92 + Some(FieldLayout::Batched) => (
93 + tensor.batched_value_at(index)?,
94 + MmField::Batched(MmBatchedField { keep_on_cpu }),
95 + ),
96 + Some(FieldLayout::Flat { sizes_key }) => {
97 + let sizes = tensors.get(sizes_key).ok_or_else(|| {
98 + multimodal!("flat tensor sizes key `{sizes_key}` is missing")
99 + })?;
100 + let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?;
101 + (
102 + tensor.flat_value_range(start, end)?,
103 + MmField::Flat(MmFlatField {
104 + slices: vec![MmSlice::Slice(SliceSpec {
105 + start: Some(0),
106 + stop: Some((end - start) as isize),
107 + step: None,
108 + })],
109 + dim: 0,
110 + keep_on_cpu,
111 + }),
112 + )
113 + }
114 + None => (
115 + tensor.clone(),
116 + MmField::Shared(MmSharedField {
117 + batch_size: len,
118 + keep_on_cpu,
119 + }),
120 + ),
121 + };
122 +
123 + data.insert(
124 + key.clone(),
125 + MmFieldElem {
126 + data: Some(value.try_into()?),
127 + field,
128 + },
129 + );
130 + }
131 +
132 + items.push(PreparedItem {
133 + data,
134 + hash: frame.hash.clone(),
135 + uuid,
136 + });
137 + }
138 +
139 + Ok(items)
140 + }
141 +}
rust/src/chat/src/multimodal/tensor.rs
+32 -14
@@ -1,7 +1,7 @@
1 use std::collections::HashMap;
2
3 use half::{bf16, f16};
4 -use llm_multimodal::{ModelSpecificValue, PreprocessedEncoderInputs as PreprocessedImages};
4 +use llm_multimodal::{ModelSpecificValue, PreprocessedEncoderInputs};
5 use vllm_engine_core_client::protocol::dtype::ModelDtype;
6 use vllm_engine_core_client::protocol::multimodal::MmKwargValue as ProtocolKwargValue;
7 use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireTensor};
@@ -25,25 +25,31 @@ pub(super) enum KwargValue {
25 Passthrough(ProtocolKwargValue),
26 }
27
28 -/// Collect `pixel_values` and model-specific outputs into one tensor map.
28 +/// Collect the primary encoder input and model-specific outputs into one
29 +/// tensor map.
30 +///
31 +/// `primary_key` names the encoder-input tensor as the model's forward kwargs
32 +/// expect it (e.g. `pixel_values` for images, `pixel_values_videos` for
33 +/// videos).
34 pub(super) fn collect_tensors(
30 - preprocessed: PreprocessedImages,
35 + preprocessed: PreprocessedEncoderInputs,
36 + primary_key: &str,
37 float_dtype: ModelDtype,
38 ) -> Result<HashMap<String, KwargValue>> {
33 - let PreprocessedImages {
39 + let PreprocessedEncoderInputs {
40 encoder_input,
41 model_specific,
42 ..
43 } = preprocessed;
44
39 - let pixel_values = {
45 + let primary_value = {
46 let shape = encoder_input.shape().to_vec();
47 let data = encoder_input.into_iter().collect();
48 KwargValue::from_f32_tensor(data, shape, float_dtype)?
49 };
50
51 let mut tensors = HashMap::new();
46 - tensors.insert("pixel_values".to_string(), pixel_values);
52 + tensors.insert(primary_key.to_string(), primary_value);
53 for (key, value) in model_specific {
54 tensors.insert(key, KwargValue::from_model_specific(value, float_dtype)?);
55 }
@@ -124,10 +130,22 @@ impl TryFrom<KwargValue> for ProtocolKwargValue {
130 }
131
132 impl KwargValue {
127 - /// Extract one image from a batched tensor field.
133 + /// First-axis length for tensor values; `None` for passthrough kwargs.
134 + pub(super) fn first_dim(&self) -> Option<usize> {
135 + match self {
136 + Self::F32Tensor { shape, .. }
137 + | Self::F16Tensor { shape, .. }
138 + | Self::Bf16Tensor { shape, .. }
139 + | Self::I64Tensor { shape, .. }
140 + | Self::U32Tensor { shape, .. } => shape.first().copied(),
141 + Self::Passthrough(_) => None,
142 + }
143 + }
144 +
145 + /// Extract one media item from a batched tensor field.
146 ///
129 - /// Batched fields use their first axis as image index and drop that axis in
130 - /// the per-feature value, matching vLLM's batched-field semantics.
147 + /// Batched fields use their first axis as media-item index and drop that
148 + /// axis in the per-feature value, matching vLLM's batched-field semantics.
149 pub(super) fn batched_value_at(&self, index: usize) -> Result<Self> {
150 match self {
151 Self::F32Tensor { data, shape } => {
@@ -154,9 +172,9 @@ impl KwargValue {
172 }
173 }
174
157 - /// Extract one image's variable-length range from a flat tensor field.
175 + /// Extract one media item's variable-length range from a flat tensor field.
176 ///
159 - /// Flat fields keep the first axis as the sliced length for this image.
177 + /// Flat fields keep the first axis as the sliced length for this item.
178 pub(super) fn flat_value_range(&self, start: usize, end: usize) -> Result<Self> {
179 match self {
180 Self::F32Tensor { data, shape } => {
@@ -184,10 +202,10 @@ impl KwargValue {
202 }
203 }
204
187 -/// Compute the first-axis range for one image in a flat tensor.
205 +/// Compute the first-axis range for one media item in a flat tensor.
206 ///
207 /// `sizes_key` names a companion tensor whose entries are cumulative slice
190 -/// sizes per image.
208 +/// sizes per media item.
209 pub(super) fn flat_range_for_index(
210 sizes: &KwargValue,
211 sizes_key: &str,
@@ -195,7 +213,7 @@ pub(super) fn flat_range_for_index(
213 ) -> Result<(usize, usize)> {
214 let sizes = tensor_as_usize_vec(sizes)?;
215 let size = *sizes.get(index).ok_or_else(|| {
198 - multimodal!("flat tensor sizes key `{sizes_key}` has no entry for image {index}")
216 + multimodal!("flat tensor sizes key `{sizes_key}` has no entry for media item {index}")
217 })?;
218 let start = sizes[..index].iter().sum::<usize>();
219 Ok((start, start + size))
rust/src/chat/src/multimodal/video.rs new
+316
@@ -0,0 +1,316 @@
1 +//! Video-modality preparation: per-clip preprocessing, config resolution,
2 +//! and per-item feature build.
3 +
4 +use std::sync::Arc;
5 +
6 +use itertools::izip;
7 +use llm_multimodal::{FieldLayout, Modality, PreprocessedEncoderInputs, VideoClip};
8 +use thiserror_ext::AsReport as _;
9 +use tracing::warn;
10 +use vllm_engine_core_client::protocol::dtype::ModelDtype;
11 +use vllm_engine_core_client::protocol::multimodal::{
12 + MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice,
13 + SliceSpec,
14 +};
15 +
16 +use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
17 +use crate::error::{Error, Result, bail_multimodal, multimodal};
18 +
19 +/// Forward-kwargs name of the primary video encoder input.
20 +///
21 +/// Video-capable vLLM models read `pixel_values_videos` alongside
22 +/// `video_grid_thw`, mirroring the HF processor output naming.
23 +const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos";
24 +
25 +impl MultimodalModelInfo {
26 + /// Preprocess fetched video clips one at a time and build per-item
27 + /// features.
28 + ///
29 + /// Unlike images, each clip runs through the preprocessor independently
30 + /// (a batch of one), so its tensors are complete per item and need no
31 + /// cross-item slicing.
32 + pub(super) async fn prepare_videos(
33 + &self,
34 + clips: Vec<Arc<VideoClip>>,
35 + uuids: Vec<Option<String>>,
36 + model_dtype: ModelDtype,
37 + ) -> Result<PreparedMedia> {
38 + let support = self.video.as_ref().ok_or_else(|| Error::UnsupportedModality {
39 + modality: Modality::Video.to_string(),
40 + })?;
41 + let mut replacements = Vec::with_capacity(clips.len());
42 + let mut items = Vec::with_capacity(clips.len());
43 +
44 + for (clip, uuid) in izip!(&clips, uuids) {
45 + let preprocessed = self.preprocess_video_clip(support, Arc::clone(clip)).await?;
46 + let mut clip_replacements =
47 + self.spec
48 + .prompt_replacements_for(&self.context, &preprocessed, Modality::Video)?;
49 + if clip_replacements.len() != 1 {
50 + bail_multimodal!(
51 + "expected exactly one prompt replacement per video clip, got {}",
52 + clip_replacements.len()
53 + );
54 + }
55 + replacements.push(clip_replacements.pop().unwrap());
56 + items.push(self.build_video_item(
57 + preprocessed,
58 + clip.hash.clone(),
59 + uuid,
60 + model_dtype,
61 + )?);
62 + }
63 +
64 + Ok(PreparedMedia {
65 + modality: Modality::Video,
66 + placeholder: support.placeholder.clone(),
67 + replacements,
68 + items,
69 + })
70 + }
71 +
72 + /// Preprocess one decoded video clip with the model's resolved vision
73 + /// processor.
74 + async fn preprocess_video_clip(
75 + &self,
76 + support: &ModalitySupport,
77 + clip: Arc<VideoClip>,
78 + ) -> Result<PreprocessedEncoderInputs> {
79 + let config = support.config.clone();
80 + let processor = support.processor;
81 +
82 + tokio::task::spawn_blocking(move || {
83 + // Prefer the borrowed-RGB fast path, which avoids materializing a
84 + // `DynamicImage` per sampled frame after media decode.
85 + if let Some(rgb_video) = clip.rgb_video() {
86 + match rgb_video.frame_refs() {
87 + Ok(frame_refs) => match processor.preprocess_video_rgb(&frame_refs, &config) {
88 + Ok(preprocessed) => return Ok(preprocessed),
89 + Err(error) => warn!(
90 + error = %error.as_report(),
91 + "RGB video preprocessing fast path failed; falling back to materialized frames"
92 + ),
93 + },
94 + Err(error) => warn!(
95 + error,
96 + "RGB video frame refs are invalid; falling back to materialized frames"
97 + ),
98 + }
99 + }
100 +
101 + let frames = clip.materialized_frames().map_err(|error| multimodal!("{error}"))?;
102 + Ok(processor.preprocess_video(&frames, &config)?)
103 + })
104 + .await
105 + .map_err(|error| multimodal!("video preprocessing task failed: {error}"))?
106 + }
107 +
108 + /// Convert one preprocessed video clip into engine kwargs.
109 + ///
110 + /// The clip is a batch of one, so no per-item slicing is required: the
111 + /// primary tensor ships as a full-range flat field (the engine re-batches
112 + /// flat fields by concatenating along the declared dim, matching vLLM's
113 + /// `flat_from_sizes` treatment of video patches), and batched metadata
114 + /// tensors drop their singleton batch axis.
115 + fn build_video_item(
116 + &self,
117 + preprocessed: PreprocessedEncoderInputs,
118 + hash: String,
119 + uuid: Option<String>,
120 + model_dtype: ModelDtype,
121 + ) -> Result<PreparedItem> {
122 + let tensors = tensor::collect_tensors(preprocessed, VIDEO_PRIMARY_KEY, model_dtype)?;
123 +
124 + let mut data = MmKwargsItem::new();
125 + for (key, tensor) in tensors {
126 + let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(&key);
127 + let (value, field) = if key == VIDEO_PRIMARY_KEY {
128 + let len = tensor
129 + .first_dim()
130 + .ok_or_else(|| multimodal!("video encoder input `{key}` is not a tensor"))?;
131 + (
132 + tensor,
133 + MmField::Flat(MmFlatField {
134 + slices: vec![MmSlice::Slice(SliceSpec {
135 + start: Some(0),
136 + stop: Some(len as isize),
137 + step: None,
138 + })],
139 + dim: 0,
140 + keep_on_cpu,
141 + }),
142 + )
143 + } else if matches!(
144 + self.spec.field_layouts.get(&key),
145 + Some(FieldLayout::Batched)
146 + ) {
147 + (
148 + tensor.batched_value_at(0)?,
149 + MmField::Batched(MmBatchedField { keep_on_cpu }),
150 + )
151 + } else {
152 + (
153 + tensor,
154 + MmField::Shared(MmSharedField {
155 + batch_size: 1,
156 + keep_on_cpu,
157 + }),
158 + )
159 + };
160 +
161 + data.insert(
162 + key,
163 + MmFieldElem {
164 + data: Some(value.try_into()?),
165 + field,
166 + },
167 + );
168 + }
169 +
170 + Ok(PreparedItem { data, hash, uuid })
171 + }
172 +}
173 +
174 +#[cfg(test)]
175 +mod tests {
176 + use std::collections::HashMap;
177 + use std::sync::Arc;
178 +
179 + use llm_multimodal::ModelSpecificValue;
180 + use ndarray::ArrayD;
181 + use vllm_engine_core_client::protocol::multimodal::MmKwargValue;
182 +
183 + use super::super::tests::{
184 + QWEN3_IMAGE_PAD_ID, QWEN3_VIDEO_PAD_ID, qwen3_vl_info, qwen3_vl_tokenizer,
185 + };
186 + use super::super::{MultimodalConfigFiles, MultimodalModelInfo};
187 + use super::*;
188 +
189 + #[test]
190 + fn from_paths_resolves_video_config_from_dedicated_file_or_processor_config() {
191 + let dir = tempfile::tempdir().unwrap();
192 + let config_path = dir.path().join("config.json");
193 + std::fs::write(
194 + &config_path,
195 + serde_json::json!({
196 + "model_type": "qwen3_vl",
197 + "image_token_id": QWEN3_IMAGE_PAD_ID,
198 + "video_token_id": QWEN3_VIDEO_PAD_ID,
199 + })
200 + .to_string(),
201 + )
202 + .unwrap();
203 +
204 + let info_for = |files: MultimodalConfigFiles<'_>| {
205 + MultimodalModelInfo::from_paths(
206 + "qwen3-vl-test".to_string(),
207 + Some("qwen3_vl".to_string()),
208 + files,
209 + Arc::new(qwen3_vl_tokenizer()),
210 + )
211 + };
212 +
213 + // Dedicated video preprocessor config file.
214 + let video_config_path = dir.path().join("video_preprocessor_config.json");
215 + std::fs::write(&video_config_path, r#"{"size":{"shortest_edge":128}}"#).unwrap();
216 + let info = info_for(MultimodalConfigFiles {
217 + config: Some(&config_path),
218 + video_preprocessor_config: Some(&video_config_path),
219 + ..Default::default()
220 + })
221 + .unwrap()
222 + .unwrap();
223 + assert!(info.video.is_some());
224 +
225 + // `video_processor` section of the combined processor config.
226 + let processor_config_path = dir.path().join("processor_config.json");
227 + std::fs::write(
228 + &processor_config_path,
229 + r#"{"video_processor":{"size":{"shortest_edge":128}}}"#,
230 + )
231 + .unwrap();
232 + let info = info_for(MultimodalConfigFiles {
233 + config: Some(&config_path),
234 + processor_config: Some(&processor_config_path),
235 + ..Default::default()
236 + })
237 + .unwrap()
238 + .unwrap();
239 + assert!(info.video.is_some());
240 +
241 + // Neither source: video support still resolves on the image config.
242 + let info = info_for(MultimodalConfigFiles {
243 + config: Some(&config_path),
244 + ..Default::default()
245 + })
246 + .unwrap()
247 + .unwrap();
248 + assert!(info.video.is_some());
249 +
250 + // Malformed dedicated file is a real error, not a silent fallback.
251 + std::fs::write(&video_config_path, r#"{"size""#).unwrap();
252 + let error = match info_for(MultimodalConfigFiles {
253 + config: Some(&config_path),
254 + video_preprocessor_config: Some(&video_config_path),
255 + ..Default::default()
256 + }) {
257 + Err(error) => error,
258 + Ok(_) => panic!("malformed video preprocessor config should fail"),
259 + };
260 + assert!(matches!(
261 + error,
262 + Error::Multimodal(message)
263 + if message.contains("failed to parse video_preprocessor_config.json")
264 + ));
265 + }
266 +
267 + #[test]
268 + fn build_video_item_names_primary_tensor_and_layouts() {
269 + let info = qwen3_vl_info();
270 + // One clip flattened to 6 patches with 4 features each.
271 + let preprocessed = PreprocessedEncoderInputs {
272 + encoder_input: ArrayD::zeros(vec![6, 4]),
273 + feature_token_counts: vec![6],
274 + item_sizes: vec![(32, 32)],
275 + model_specific: HashMap::from([
276 + (
277 + "video_grid_thw".to_string(),
278 + ModelSpecificValue::int_2d(vec![1, 2, 3], 1, 3),
279 + ),
280 + (
281 + "patches_per_video".to_string(),
282 + ModelSpecificValue::int_1d(vec![6]),
283 + ),
284 + ]),
285 + };
286 +
287 + let item = info
288 + .build_video_item(
289 + preprocessed,
290 + "<hash>".to_string(),
291 + None,
292 + ModelDtype::Float32,
293 + )
294 + .unwrap();
295 +
296 + let primary = &item.data[VIDEO_PRIMARY_KEY];
297 + assert!(matches!(
298 + &primary.field,
299 + MmField::Flat(MmFlatField { slices, dim: 0, .. })
300 + if matches!(
301 + slices.as_slice(),
302 + [MmSlice::Slice(SliceSpec { start: Some(0), stop: Some(6), step: None })]
303 + )
304 + ));
305 +
306 + // Batched metadata drops its singleton batch axis per item.
307 + let grid = &item.data["video_grid_thw"];
308 + assert!(matches!(&grid.field, MmField::Batched(_)));
309 + let MmKwargValue::Tensor(grid_tensor) = grid.data.as_ref().unwrap() else {
310 + panic!("expected tensor value for video_grid_thw");
311 + };
312 + assert_eq!(grid_tensor.shape, vec![3]);
313 +
314 + assert_eq!(item.hash, "<hash>");
315 + }
316 +}
rust/src/chat/src/renderer/hf/mod.rs
+83 -8
@@ -31,9 +31,14 @@ pub use template::{load_chat_template, resolve_chat_template};
31
32 pub use self::format::ChatTemplateContentFormatOption;
33
34 -#[derive(Debug, Clone)]
34 +/// Template-visible placeholder tokens per supported modality.
35 +///
36 +/// A `None` token means the loaded model does not support that modality, and
37 +/// content parts of that modality are rejected during rendering.
38 +#[derive(Debug, Clone, Default)]
39 pub struct MultimodalRenderInfo {
36 - pub placeholder_token: String,
40 + pub image_token: Option<String>,
41 + pub video_token: Option<String>,
42 }
43
44 /// Hugging Face chat-template renderer backed by the local Jinja chat-template
@@ -254,6 +259,7 @@ enum TemplateContent {
259 enum TemplateContentPart {
260 Text { text: String },
261 Image,
262 + Video,
263 }
264
265 #[derive(Debug, Serialize)]
@@ -417,9 +423,17 @@ fn to_template_openai_content(
423 }
424 // All multimodal contents are normalized to `{ "type": <modality> }`.
425 ChatContentPart::ImageUrl { .. } => {
420 - multimodal.ok_or(Error::UnsupportedMultimodalContent("image_url"))?;
426 + multimodal
427 + .and_then(|multimodal| multimodal.image_token.as_ref())
428 + .ok_or(Error::UnsupportedMultimodalContent("image_url"))?;
429 Ok(TemplateContentPart::Image)
430 }
431 + ChatContentPart::VideoUrl { .. } => {
432 + multimodal
433 + .and_then(|multimodal| multimodal.video_token.as_ref())
434 + .ok_or(Error::UnsupportedMultimodalContent("video_url"))?;
435 + Ok(TemplateContentPart::Video)
436 + }
437 })
438 .collect(),
439 }
@@ -437,9 +451,16 @@ fn to_template_string_content(
451 match part {
452 ChatContentPart::Text { text } => out.push_str(text),
453 ChatContentPart::ImageUrl { .. } => {
440 - let multimodal =
441 - multimodal.ok_or(Error::UnsupportedMultimodalContent("image_url"))?;
442 - out.push_str(&multimodal.placeholder_token);
454 + let image_token = multimodal
455 + .and_then(|multimodal| multimodal.image_token.as_ref())
456 + .ok_or(Error::UnsupportedMultimodalContent("image_url"))?;
457 + out.push_str(image_token);
458 + }
459 + ChatContentPart::VideoUrl { .. } => {
460 + let video_token = multimodal
461 + .and_then(|multimodal| multimodal.video_token.as_ref())
462 + .ok_or(Error::UnsupportedMultimodalContent("video_url"))?;
463 + out.push_str(video_token);
464 }
465 }
466 }
@@ -468,7 +489,7 @@ fn append_continue_final_message_tag(message: &mut TemplateMessage) -> Result<St
489 // Pick the last text part in the message.
490 TemplateContent::OpenAi(parts) => parts.iter_mut().rev().find_map(|part| match part {
491 TemplateContentPart::Text { text } => Some(text),
471 - TemplateContentPart::Image => None,
492 + TemplateContentPart::Image | TemplateContentPart::Video => None,
493 }),
494 };
495 let text = text.ok_or_else(|| {
@@ -577,7 +598,8 @@ mod tests {
598 ) -> Result<crate::RenderedPrompt> {
599 HfChatRenderer::new(Some(template.to_string()), HashMap::new(), content_format)?
600 .with_multimodal(Some(MultimodalRenderInfo {
580 - placeholder_token: "<image>".to_string(),
601 + image_token: Some("<image>".to_string()),
602 + video_token: Some("<video>".to_string()),
603 }))
604 .render(request)
605 }
@@ -590,6 +612,14 @@ mod tests {
612 ])])
613 }
614
615 + fn video_request() -> ChatRequest {
616 + sample_request(vec![ChatMessage::user(vec![
617 + ChatContentPart::text("a"),
618 + ChatContentPart::video_url("https://example.com/demo.mp4"),
619 + ChatContentPart::text("b"),
620 + ])])
621 + }
622 +
623 #[test]
624 fn string_content_format_replaces_image_with_placeholder_text() {
625 let rendered = render_mm(
@@ -602,6 +632,18 @@ mod tests {
632 assert_eq!(rendered.prompt, Prompt::Text("a<image>b".to_string()));
633 }
634
635 + #[test]
636 + fn string_content_format_replaces_video_with_placeholder_text() {
637 + let rendered = render_mm(
638 + "{{ messages[0].content }}",
639 + &video_request(),
640 + ChatTemplateContentFormatOption::String,
641 + )
642 + .unwrap();
643 +
644 + assert_eq!(rendered.prompt, Prompt::Text("a<video>b".to_string()));
645 + }
646 +
647 #[test]
648 fn openai_content_format_normalizes_image_url_for_template() {
649 let rendered = render_mm(
@@ -614,6 +656,39 @@ mod tests {
656 assert_eq!(rendered.prompt, Prompt::Text("a<|image_pad|>b".to_string()));
657 }
658
659 + #[test]
660 + fn openai_content_format_normalizes_video_url_for_template() {
661 + let rendered = render_mm(
662 + "{% for item in messages[0].content %}{% if item.type == 'video' %}<|video_pad|>{% else %}{{ item.text }}{% endif %}{% endfor %}",
663 + &video_request(),
664 + ChatTemplateContentFormatOption::OpenAi,
665 + )
666 + .unwrap();
667 +
668 + assert_eq!(rendered.prompt, Prompt::Text("a<|video_pad|>b".to_string()));
669 + }
670 +
671 + #[test]
672 + fn video_parts_are_rejected_when_model_lacks_video_support() {
673 + let error = HfChatRenderer::new(
674 + Some("{{ messages[0].content }}".to_string()),
675 + HashMap::new(),
676 + ChatTemplateContentFormatOption::String,
677 + )
678 + .unwrap()
679 + .with_multimodal(Some(MultimodalRenderInfo {
680 + image_token: Some("<image>".to_string()),
681 + video_token: None,
682 + }))
683 + .render(&video_request())
684 + .unwrap_err();
685 +
686 + assert!(matches!(
687 + error,
688 + Error::UnsupportedMultimodalContent("video_url")
689 + ));
690 + }
691 +
692 #[test]
693 fn chat_template_supports_pycompat_templates() {
694 let request = sample_request(vec![ChatMessage::text(ChatRole::User, "<think>hello")]);
rust/src/chat/src/request.rs
+36 -1
@@ -36,7 +36,13 @@ pub enum ChatContentPart {
36 detail: Option<ImageDetail>,
37 uuid: Option<String>,
38 },
39 + /// One video URL/data URL content block.
40 + VideoUrl {
41 + video_url: String,
42 + uuid: Option<String>,
43 + },
44 // ImageData...
45 + // VideoData...
46 // ImageEmbeds...
47 }
48
@@ -55,12 +61,21 @@ impl ChatContentPart {
61 }
62 }
63
64 + /// Construct one video URL content part with the given URL string.
65 + pub fn video_url(video_url: impl Into<String>) -> Self {
66 + Self::VideoUrl {
67 + video_url: video_url.into(),
68 + uuid: None,
69 + }
70 + }
71 +
72 /// Return the text content of this part when it's a text block, or an
73 /// "unsupported multimodal content" error otherwise.
74 pub(crate) fn as_text(&self) -> Result<&str> {
75 match self {
76 Self::Text { text } => Ok(text),
77 Self::ImageUrl { .. } => Err(Error::UnsupportedMultimodalContent("image_url")),
78 + Self::VideoUrl { .. } => Err(Error::UnsupportedMultimodalContent("video_url")),
79 }
80 }
81
@@ -73,7 +88,7 @@ impl ChatContentPart {
88 pub(crate) fn is_multimodal(&self) -> bool {
89 match self {
90 Self::Text { .. } => false,
76 - Self::ImageUrl { .. } => true,
91 + Self::ImageUrl { .. } | Self::VideoUrl { .. } => true,
92 }
93 }
94 }
@@ -555,6 +570,26 @@ mod tests {
570 assert_eq!(content, ChatContent::Text("hello".to_string()));
571 }
572
573 + #[test]
574 + fn chat_content_video_url_part_round_trips_through_serde() {
575 + let content = ChatContent::Parts(vec![ChatContentPart::VideoUrl {
576 + video_url: "https://example.com/demo.mp4".to_string(),
577 + uuid: Some("video-1".to_string()),
578 + }]);
579 +
580 + let value = to_value(&content).unwrap();
581 + assert_eq!(
582 + value,
583 + json!([{
584 + "type": "video_url",
585 + "video_url": "https://example.com/demo.mp4",
586 + "uuid": "video-1",
587 + }])
588 + );
589 + let decoded: ChatContent = serde_json::from_value(value).unwrap();
590 + assert_eq!(decoded, content);
591 + }
592 +
593 #[test]
594 fn chat_content_deserializes_from_openai_text_blocks() {
595 let content: ChatContent =
rust/src/server/src/routes/openai/chat_completions/convert.rs
+15 -6
@@ -290,7 +290,10 @@ fn convert_content(content: MessageContent) -> Result<ChatContent, ApiError> {
290 detail: image_url.detail,
291 uuid,
292 }),
293 - _ => bail_invalid_request!("Only text and image_url content parts are supported."),
293 + ContentPart::VideoUrl { video_url, uuid } => Ok(ChatContentPart::VideoUrl {
294 + video_url: video_url.url,
295 + uuid,
296 + }),
297 })
298 .try_collect()
299 .map(ChatContent::Parts),
@@ -772,28 +775,34 @@ mod tests {
775 }
776
777 #[test]
775 - fn prepare_chat_request_rejects_video_content_parts() {
778 + fn prepare_chat_request_accepts_video_content_parts() {
779 let request = ChatCompletionRequest {
780 messages: vec![ChatMessage::User {
781 content: MessageContent::Parts(vec![ContentPart::VideoUrl {
782 video_url: VideoUrl {
783 url: "https://example.com/video.mp4".to_string(),
784 },
785 + uuid: Some("video-uuid".to_string()),
786 }]),
787 name: None,
788 }],
789 ..base_request()
790 };
791
788 - let error = prepare_chat_request(
792 + let prepared = prepare_chat_request(
793 request,
794 &served(&["Qwen/Qwen1.5-0.5B-Chat"]),
795 ResolvedRequestContext::default(),
796 )
793 - .unwrap_err();
797 + .expect("request is valid");
798
795 - expect!["Only text and image_url content parts are supported."]
796 - .assert_eq(&error.to_error_response().error.message);
799 + assert_eq!(
800 + prepared.chat_request.messages,
801 + vec![VllmChatMessage::user(vec![ChatContentPart::VideoUrl {
802 + video_url: "https://example.com/video.mp4".to_string(),
803 + uuid: Some("video-uuid".to_string()),
804 + }])]
805 + );
806 }
807
808 #[test]
rust/src/server/src/routes/openai/utils/types.rs
+5 -1
@@ -91,7 +91,11 @@ pub enum ContentPart {
91 uuid: Option<String>,
92 },
93 #[serde(rename = "video_url")]
94 - VideoUrl { video_url: VideoUrl },
94 + VideoUrl {
95 + video_url: VideoUrl,
96 + #[serde(skip_serializing_if = "Option::is_none")]
97 + uuid: Option<String>,
98 + },
99 }
100
101 #[serde_with::skip_serializing_none]
rust/src/server/src/routes/tests.rs
+8 -4
@@ -504,7 +504,7 @@ impl ChatRenderer for FakeChatBackend {
504 let placeholder = self
505 .multimodal_model_info
506 .as_ref()
507 - .map(|info| info.placeholder_token())
507 + .and_then(|info| info.placeholder_token(llm_multimodal::Modality::Image))
508 .unwrap_or("<image>");
509 let mut prompt = String::new();
510 for message in &request.messages {
@@ -544,7 +544,9 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
544 for part in parts {
545 match part {
546 ChatContentPart::Text { text } => out.push_str(text),
547 - ChatContentPart::ImageUrl { .. } => out.push_str(placeholder),
547 + ChatContentPart::ImageUrl { .. } | ChatContentPart::VideoUrl { .. } => {
548 + out.push_str(placeholder)
549 + }
550 }
551 }
552 out
@@ -565,8 +567,10 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
567 let info = vllm_chat::multimodal::MultimodalModelInfo::from_paths(
568 "qwen2-vl-test".to_string(),
569 Some("qwen2_vl".to_string()),
568 - Some(&config_path),
569 - None,
570 + vllm_chat::multimodal::MultimodalConfigFiles {
571 + config: Some(&config_path),
572 + ..Default::default()
573 + },
574 Arc::new(fake_chat_tokenizer()),
575 )
576 .expect("load multimodal info")
rust/src/text/src/backend/hf/model_files.rs
+19
@@ -41,6 +41,10 @@ pub struct ResolvedModelFiles {
41 pub tokenizer_config_path: Option<PathBuf>,
42 pub generation_config_path: Option<PathBuf>,
43 pub preprocessor_config_path: Option<PathBuf>,
44 + /// Video-specific preprocessor config, when provided by the model repo.
45 + pub video_preprocessor_config_path: Option<PathBuf>,
46 + /// Combined processor config, which may embed a `video_processor` section.
47 + pub processor_config_path: Option<PathBuf>,
48 pub chat_template_path: Option<PathBuf>,
49 pub config_path: Option<PathBuf>,
50 }
@@ -70,6 +74,11 @@ fn resolve_local_model_files(model_dir: &Path) -> Result<ResolvedModelFiles> {
74 tokenizer_config_path,
75 generation_config_path: local_file_if_exists(model_dir, "generation_config.json"),
76 preprocessor_config_path: local_file_if_exists(model_dir, "preprocessor_config.json"),
77 + video_preprocessor_config_path: local_file_if_exists(
78 + model_dir,
79 + "video_preprocessor_config.json",
80 + ),
81 + processor_config_path: local_file_if_exists(model_dir, "processor_config.json"),
82 chat_template_path: discover_chat_template_in_dir(model_dir),
83 config_path: local_file_if_exists(model_dir, "config.json"),
84 })
@@ -107,6 +116,10 @@ async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles
116 download_if_present(&repo, model_id, &siblings, "generation_config.json").await?;
117 let preprocessor_config_path =
118 download_if_present(&repo, model_id, &siblings, "preprocessor_config.json").await?;
119 + let video_preprocessor_config_path =
120 + download_if_present(&repo, model_id, &siblings, "video_preprocessor_config.json").await?;
121 + let processor_config_path =
122 + download_if_present(&repo, model_id, &siblings, "processor_config.json").await?;
123 let chat_template_name = siblings
124 .contains("chat_template.json")
125 .then_some("chat_template.json")
@@ -123,6 +136,8 @@ async fn resolve_remote_model_files(model_id: &str) -> Result<ResolvedModelFiles
136 tokenizer_config_path,
137 generation_config_path,
138 preprocessor_config_path,
139 + video_preprocessor_config_path,
140 + processor_config_path,
141 chat_template_path,
142 config_path,
143 })
@@ -143,6 +158,8 @@ fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFile
158 })?;
159 let generation_config_path = cache_repo.get("generation_config.json");
160 let preprocessor_config_path = cache_repo.get("preprocessor_config.json");
161 + let video_preprocessor_config_path = cache_repo.get("video_preprocessor_config.json");
162 + let processor_config_path = cache_repo.get("processor_config.json");
163 let chat_template_path = discover_chat_template_in_dir(model_dir);
164 let config_path = cache_repo.get("config.json");
165
@@ -151,6 +168,8 @@ fn resolve_cached_model_files(model_id: &str) -> Result<Option<ResolvedModelFile
168 tokenizer_config_path,
169 generation_config_path,
170 preprocessor_config_path,
171 + video_preprocessor_config_path,
172 + processor_config_path,
173 chat_template_path,
174 config_path,
175 }))