[Rust Frontend] Construct model-owned vision processors through specs (#58109)
Co-authored-by: Codex <noreply@openai.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com>
Bugen Zhao committed
Sep 22, 2026 at 18:01 UTC
81d7293c2167e39f3ffddc9a82d633f94e8a1eaa
5 files changed
+64
-58
rust/Cargo.lock
+1
-1
@@ -2290,7 +2290,7 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
2290
[[package]]
2291
name = "llm-multimodal"
2292
version = "1.7.1"
2293
-source = "git+https://github.com/smg-project/llm-multimodal?rev=24c676dd49d73f05cb4068aade2e60397a5c0afe#24c676dd49d73f05cb4068aade2e60397a5c0afe"
2293
+source = "git+https://github.com/smg-project/llm-multimodal?rev=53ca0d6ea829af7477b6b63851cc70bed409bbc8#53ca0d6ea829af7477b6b63851cc70bed409bbc8"
2294
dependencies = [
2295
"anyhow",
2296
"base64 0.22.1",
rust/Cargo.toml
+1
-1
@@ -61,7 +61,7 @@ indexmap = "2.13.0"
61
indicatif = "0.18.4"
62
itertools = "0.14.0"
63
libc = "0.2.177"
64
-llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "24c676dd49d73f05cb4068aade2e60397a5c0afe", default-features = false, features = ["native-tls"] }
64
+llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "53ca0d6ea829af7477b6b63851cc70bed409bbc8", default-features = false, features = ["native-tls"] }
65
mimalloc = "0.1.52"
66
minijinja = { version = "2.24", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
67
minijinja-contrib = { version = "2.24", features = ["pycompat"] }
rust/src/chat/src/multimodal.rs
+52
-44
@@ -17,12 +17,12 @@ use std::path::Path;
17
use std::sync::{Arc, LazyLock};
18
19
use itertools::{Either, izip};
20
+use llm_multimodal::registry::ModelRegistryError;
21
use llm_multimodal::{
22
AsyncMultiModalTracker, AudioClip, AudioPreProcessor, EncoderFieldLayouts, FieldLayout,
23
ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata,
24
ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs,
25
PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
25
- VisionProcessorRegistry,
26
};
27
use serde::{Deserialize, Serialize};
28
use thiserror_ext::AsReport as _;
@@ -55,8 +55,8 @@ pub use self::timing::{mm_request_span, mm_timing_layer};
55
#[derive(Clone)]
56
pub struct MultimodalModelInfo {
57
context: MultimodalModelContext,
58
- image: Option<ModalitySupport>,
59
- video: Option<ModalitySupport>,
58
+ image: Option<VisionModalitySupport>,
59
+ video: Option<VisionModalitySupport>,
60
audio: Option<AudioModalitySupport>,
61
media_connector: Arc<MediaConnector>,
62
/// Maximum number of input items allowed per prompt for each modality.
@@ -133,13 +133,18 @@ impl MultimodalModelContext {
133
REGISTRY.lookup(&self.metadata())
134
}
135
136
- /// Resolve a static vision preprocessor for one loaded model.
137
- ///
138
- /// The vision preprocessor serves both the image and video modalities.
139
- fn resolve_vision_processor(&self) -> Option<&'static dyn VisionPreProcessor> {
140
- static REGISTRY: LazyLock<VisionProcessorRegistry> =
141
- LazyLock::new(VisionProcessorRegistry::with_defaults);
142
- REGISTRY.find(&self.model_id, self.model_type.as_deref())
136
+ /// Build a vision preprocessor for one loaded model and modality.
137
+ fn resolve_vision_processor(
138
+ &self,
139
+ model_spec: &'static dyn ModelProcessorSpec,
140
+ preprocessor_config: &PreProcessorConfig,
141
+ modality: Modality,
142
+ ) -> Result<Option<Arc<dyn VisionPreProcessor>>> {
143
+ match model_spec.vision_processor(&self.metadata(), preprocessor_config, modality) {
144
+ Ok(processor) => Ok(Some(Arc::from(processor))),
145
+ Err(ModelRegistryError::UnsupportedModality { .. }) => Ok(None),
146
+ Err(error) => Err(error.into()),
147
+ }
148
}
149
150
/// Resolve an audio preprocessor for one loaded model.
@@ -148,7 +153,7 @@ impl MultimodalModelContext {
153
model_spec: &'static dyn ModelProcessorSpec,
154
preprocessor_config: &PreProcessorConfig,
155
) -> Option<Arc<dyn AudioPreProcessor>> {
151
- model_spec.audio_processor(&self.config, preprocessor_config).map(Arc::from)
156
+ model_spec.audio_processor(&self.metadata(), preprocessor_config).map(Arc::from)
157
}
158
}
159
@@ -242,17 +247,16 @@ impl ResolvedPlaceholder {
247
}
248
}
249
245
-/// Static per-modality vision preprocessor plus its loaded config, resolved
246
-/// placeholder tokens, and the model's shared tensor-layout spec.
250
+/// Model-owned vision preprocessor plus resolved placeholder tokens and
251
+/// the model's shared tensor-layout spec.
252
#[derive(Clone)]
248
-struct ModalitySupport {
253
+struct VisionModalitySupport {
254
spec: ResolvedMultimodalSpec,
255
placeholder: ResolvedPlaceholder,
251
- processor: &'static dyn VisionPreProcessor,
252
- config: PreProcessorConfig,
256
+ processor: Arc<dyn VisionPreProcessor>,
257
}
258
255
-/// Static audio preprocessor plus the resolved model contract for its output.
259
+/// Model-owned audio preprocessor plus the resolved model contract for its output.
260
#[derive(Clone)]
261
struct AudioModalitySupport {
262
spec: ResolvedMultimodalSpec,
@@ -399,7 +403,7 @@ impl MultimodalModelInfo {
403
&context,
404
preprocessor_config.clone(),
405
video_preprocessor_config,
402
- );
406
+ )?;
407
let audio = Self::resolve_audio_lane(&context, &preprocessor_config)?;
408
409
if image.is_none() && video.is_none() && audio.is_none() {
@@ -430,27 +434,19 @@ impl MultimodalModelInfo {
434
///
435
/// A missing model spec or vision processor disables both lanes but is
436
/// not an error: unsupported model families are logged and skipped.
437
+ /// Processor construction errors propagate to model loading.
438
fn resolve_vision_lanes(
439
context: &MultimodalModelContext,
440
preprocessor_config: PreProcessorConfig,
441
video_preprocessor_config: PreProcessorConfig,
437
- ) -> (Option<ModalitySupport>, Option<ModalitySupport>) {
442
+ ) -> Result<(Option<VisionModalitySupport>, Option<VisionModalitySupport>)> {
443
let Some(raw_spec) = context.resolve_model_spec() else {
444
warn!(
445
model_id = context.model_id,
446
model_type = context.model_type,
447
"multimodal model spec is not registered; disabling image/video support for this model"
448
);
444
- return (None, None);
445
- };
446
-
447
- let Some(processor) = context.resolve_vision_processor() else {
448
- warn!(
449
- model_id = context.model_id,
450
- model_type = context.model_type,
451
- "vision processor is not registered; disabling image/video support for this model"
452
- );
453
- return (None, None);
449
+ return Ok((None, None));
450
};
451
452
// Warn and disable the modality if the placeholder resolution fails.
@@ -468,14 +464,19 @@ impl MultimodalModelInfo {
464
}
465
};
466
471
- let image = resolve_placeholder(Modality::Image).map(|placeholder| ModalitySupport {
472
- spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image),
473
- placeholder,
474
- processor,
475
- config: preprocessor_config.clone(),
476
- });
467
+ let image = if let Some(placeholder) = resolve_placeholder(Modality::Image) {
468
+ context
469
+ .resolve_vision_processor(raw_spec, &preprocessor_config, Modality::Image)?
470
+ .map(|processor| VisionModalitySupport {
471
+ spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image),
472
+ placeholder,
473
+ processor,
474
+ })
475
+ } else {
476
+ None
477
+ };
478
478
- let video = resolve_placeholder(Modality::Video).and_then(|placeholder| {
479
+ let video = if let Some(placeholder) = resolve_placeholder(Modality::Video) {
480
// Placeholder expansion attributes markers to modalities by token
481
// ID, so a marker shared with the image modality is ambiguous.
482
let image_marker = image.as_ref().map(|image| image.placeholder.marker_token_id);
@@ -487,16 +488,23 @@ impl MultimodalModelInfo {
488
);
489
None
490
} else {
490
- Some(ModalitySupport {
491
- spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video),
492
- placeholder,
493
- processor,
494
- config: video_preprocessor_config,
495
- })
491
+ context
492
+ .resolve_vision_processor(
493
+ raw_spec,
494
+ &video_preprocessor_config,
495
+ Modality::Video,
496
+ )?
497
+ .map(|processor| VisionModalitySupport {
498
+ spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video),
499
+ placeholder,
500
+ processor,
501
+ })
502
}
497
- });
503
+ } else {
504
+ None
505
+ };
506
499
- (image, video)
507
+ Ok((image, video))
508
}
509
510
/// Resolve a spec-backed audio lane when the model advertises audio support.
rust/src/chat/src/multimodal/image.rs
+4
-5
@@ -10,7 +10,7 @@ use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs};
10
use vllm_engine_core_client::protocol::dtype::ModelDtype;
11
12
use super::timing::MM_STAGE_TARGET;
13
-use super::{ModalitySupport, MultimodalModelInfo, PreparedMedia, item};
13
+use super::{MultimodalModelInfo, PreparedMedia, VisionModalitySupport, item};
14
use crate::error::{Error, Result, bail_multimodal, multimodal};
15
16
/// Forward-kwargs name of the primary image encoder input.
@@ -63,15 +63,14 @@ impl MultimodalModelInfo {
63
/// conversion.
64
async fn preprocess_images(
65
&self,
66
- support: &ModalitySupport,
66
+ support: &VisionModalitySupport,
67
image_frames: &[Arc<ImageFrame>],
68
) -> Result<PreprocessedEncoderInputs> {
69
- let config = support.config.clone();
70
- let processor = support.processor;
69
+ let processor = Arc::clone(&support.processor);
70
let images = image_frames.iter().map(|frame| frame.data().clone()).collect::<Vec<_>>();
71
72
// TODO: is it still necessary given that we've already in a dedicated runtime?
74
- tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images, &config)?))
73
+ tokio::task::spawn_blocking(move || Ok(processor.preprocess(&images)?))
74
.await
75
.map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
76
}
rust/src/chat/src/multimodal/video.rs
+6
-7
@@ -17,7 +17,7 @@ use vllm_engine_core_client::protocol::multimodal::{
17
};
18
19
use super::timing::MM_STAGE_TARGET;
20
-use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
20
+use super::{MultimodalModelInfo, PreparedItem, PreparedMedia, VisionModalitySupport, tensor};
21
use crate::error::{Error, Result, bail_multimodal, multimodal};
22
23
/// Forward-kwargs name of the primary video encoder input.
@@ -83,18 +83,17 @@ impl MultimodalModelInfo {
83
/// processor.
84
async fn preprocess_video_clip(
85
&self,
86
- support: &ModalitySupport,
86
+ support: &VisionModalitySupport,
87
clip: Arc<VideoClip>,
88
) -> Result<PreprocessedEncoderInputs> {
89
- let config = support.config.clone();
90
- let processor = support.processor;
89
+ let processor = Arc::clone(&support.processor);
90
91
tokio::task::spawn_blocking(move || {
92
// Prefer the borrowed-RGB fast path, which avoids materializing a
93
// `DynamicImage` per sampled frame after media decode.
94
if let Some(rgb_video) = clip.rgb_video() {
95
match rgb_video.frame_refs() {
97
- Ok(frame_refs) => match processor.preprocess_video_rgb(&frame_refs, &config) {
96
+ Ok(frame_refs) => match processor.preprocess_video_rgb(&frame_refs) {
97
Ok(preprocessed) => return Ok(preprocessed),
98
Err(error) => warn!(
99
error = %error.as_report(),
@@ -109,7 +108,7 @@ impl MultimodalModelInfo {
108
}
109
110
let frames = clip.materialized_frames().map_err(|error| multimodal!("{error}"))?;
112
- Ok(processor.preprocess_video(&frames, &config)?)
111
+ Ok(processor.preprocess_video(&frames)?)
112
})
113
.await
114
.map_err(|error| multimodal!("video preprocessing task failed: {error}"))?
@@ -124,7 +123,7 @@ impl MultimodalModelInfo {
123
/// `flat_from_sizes` treatment of video patches), and batched metadata
124
/// tensors drop their singleton batch axis.
125
fn build_video_item(
127
- support: &ModalitySupport,
126
+ support: &VisionModalitySupport,
127
preprocessed: PreprocessedEncoderInputs,
128
hash: String,
129
uuid: Option<String>,