[Rust Frontend] Bump llm-multimodal version (#47530)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Isotr0py committed
Jul 3, 2026 at 17:48 UTC
400a9c386da89ef86a72f4f84300ef94030fac6d
6 files changed
+48
-24
rust/Cargo.lock
+28
-12
@@ -489,9 +489,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
489
490
[[package]]
491
name = "bytes"
492
-version = "1.11.1"
492
+version = "1.12.0"
493
source = "registry+https://github.com/rust-lang/crates.io-index"
494
-checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
494
+checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593"
495
dependencies = [
496
"serde",
497
]
@@ -2112,6 +2112,16 @@ version = "0.2.183"
2112
source = "registry+https://github.com/rust-lang/crates.io-index"
2113
checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d"
2114
2115
+[[package]]
2116
+name = "libloading"
2117
+version = "0.8.9"
2118
+source = "registry+https://github.com/rust-lang/crates.io-index"
2119
+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
2120
+dependencies = [
2121
+ "cfg-if",
2122
+ "windows-link",
2123
+]
2124
+
2125
[[package]]
2126
name = "libm"
2127
version = "0.2.16"
@@ -2156,21 +2166,27 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
2166
2167
[[package]]
2168
name = "llm-multimodal"
2159
-version = "1.5.0"
2160
-source = "git+https://github.com/vllm-project/llm-multimodal?rev=046b669bd1c4faa2a7e05344d8cbf7b2befb37d5#046b669bd1c4faa2a7e05344d8cbf7b2befb37d5"
2169
+version = "1.7.1"
2170
+source = "git+https://github.com/smg-project/llm-multimodal?rev=7d74582aeaf0e4086a44964382655d22f1af0686#7d74582aeaf0e4086a44964382655d22f1af0686"
2171
dependencies = [
2172
+ "anyhow",
2173
"base64 0.22.1",
2174
"blake3",
2175
"bytes",
2176
"fast_image_resize",
2177
+ "hf-hub",
2178
"image",
2179
+ "libloading",
2180
"ndarray 0.17.2",
2181
"once_cell",
2182
"reqwest",
2183
"serde",
2184
"serde_json",
2185
+ "serde_with",
2186
+ "tempfile",
2187
"thiserror 2.0.18",
2188
"tokio",
2189
+ "tracing",
2190
"url",
2191
]
2192
@@ -2365,9 +2381,9 @@ dependencies = [
2381
2382
[[package]]
2383
name = "mio"
2368
-version = "1.1.1"
2384
+version = "1.2.1"
2385
source = "registry+https://github.com/rust-lang/crates.io-index"
2370
-checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc"
2386
+checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda"
2387
dependencies = [
2388
"libc",
2389
"wasi",
@@ -2532,9 +2548,9 @@ dependencies = [
2548
2549
[[package]]
2550
name = "once_cell"
2535
-version = "1.21.3"
2551
+version = "1.21.4"
2552
source = "registry+https://github.com/rust-lang/crates.io-index"
2537
-checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
2553
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
2554
2555
[[package]]
2556
name = "once_cell_polyfill"
@@ -4452,9 +4468,9 @@ dependencies = [
4468
4469
[[package]]
4470
name = "tokio"
4455
-version = "1.50.0"
4471
+version = "1.52.3"
4472
source = "registry+https://github.com/rust-lang/crates.io-index"
4457
-checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d"
4473
+checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
4474
dependencies = [
4475
"bytes",
4476
"libc",
@@ -4469,9 +4485,9 @@ dependencies = [
4485
4486
[[package]]
4487
name = "tokio-macros"
4472
-version = "2.6.1"
4488
+version = "2.7.0"
4489
source = "registry+https://github.com/rust-lang/crates.io-index"
4474
-checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c"
4490
+checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
4491
dependencies = [
4492
"proc-macro2",
4493
"quote",
rust/Cargo.toml
+2
-2
@@ -31,7 +31,7 @@ axum = "0.8.8"
31
base64 = "0.22.1"
32
bytemuck = { version = "1.25.0", features = ["extern_crate_alloc"] }
33
byteorder = "1.5.0"
34
-bytes = "1.11.1"
34
+bytes = "1.12.0"
35
clap = { version = "4.5.38", features = ["derive", "env"] }
36
criterion = "0.5.1"
37
easy-ext = "1.0.3"
@@ -53,7 +53,7 @@ hyper-util = { version = "0.1.20", features = [
53
indexmap = "2.13.0"
54
itertools = "0.14.0"
55
libc = "0.2.177"
56
-llm-multimodal = { git = "https://github.com/vllm-project/llm-multimodal", rev = "046b669bd1c4faa2a7e05344d8cbf7b2befb37d5" }
56
+llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "7d74582aeaf0e4086a44964382655d22f1af0686" }
57
mimalloc = "0.1.52"
58
minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
59
minijinja-contrib = { version = "2.0", features = ["pycompat"] }
rust/src/chat/src/backend/hf.rs
+4
-1
@@ -303,7 +303,7 @@ mod tests {
303
.unwrap()
304
.join("preprocessor_config.json");
305
write_json(&preprocessor_config_path, r#"{"size":[672,672]}"#);
306
- files.preprocessor_config_path = Some(preprocessor_config_path);
306
+ files.preprocessor_config_path = Some(preprocessor_config_path.clone());
307
308
let backend = HfChatBackend::from_resolved_model_files(
309
files.clone(),
@@ -321,6 +321,9 @@ mod tests {
321
322
assert!(backend.multimodal_model_info().is_none());
323
324
+ let invalid_preprocessor_config = r#"{"size":[672,672]"#;
325
+ write_json(&preprocessor_config_path, invalid_preprocessor_config);
326
+
327
let error = HfChatBackend::from_resolved_model_files(
328
files,
329
"test-model".to_string(),
rust/src/chat/src/multimodal.rs
+9
-4
@@ -16,10 +16,11 @@ use std::sync::{Arc, LazyLock};
16
17
use itertools::izip;
18
use llm_multimodal::{
19
- AsyncMultiModalTracker, FieldLayout, ImagePreProcessor, ImageProcessorRegistry, MediaConnector,
20
- MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec,
21
- ModelRegistry, PreProcessorConfig, PreprocessedImages, PromptReplacement, TokenResolver,
22
- TrackedMedia,
19
+ AsyncMultiModalTracker, FieldLayout, MediaConnector, MediaConnectorConfig, MediaContentPart,
20
+ Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry, PreProcessorConfig,
21
+ PreprocessedEncoderInputs as PreprocessedImages, PromptReplacement, Tokenizer as TokenResolver,
22
+ TrackedMedia, VisionPreProcessor as ImagePreProcessor,
23
+ VisionProcessorRegistry as ImageProcessorRegistry,
24
};
25
use tracing::warn;
26
use vllm_engine_core_client::protocol::dtype::ModelDtype;
@@ -555,6 +556,10 @@ impl TokenResolver for TokenizerResolver {
556
fn id_to_token(&self, id: u32) -> Option<String> {
557
self.0.id_to_token(id)
558
}
559
+
560
+ fn encode_text(&self, text: &str) -> Option<Vec<u32>> {
561
+ self.0.encode(text, false).ok()
562
+ }
563
}
564
565
#[cfg(test)]
rust/src/chat/src/multimodal/tensor.rs
+4
-4
@@ -1,7 +1,7 @@
1
use std::collections::HashMap;
2
3
use half::{bf16, f16};
4
-use llm_multimodal::{ModelSpecificValue, PreprocessedImages};
4
+use llm_multimodal::{ModelSpecificValue, PreprocessedEncoderInputs as PreprocessedImages};
5
use vllm_engine_core_client::protocol::dtype::ModelDtype;
6
use vllm_engine_core_client::protocol::multimodal::MmKwargValue as ProtocolKwargValue;
7
use vllm_engine_core_client::protocol::tensor::{ShapeExt as _, WireTensor};
@@ -31,14 +31,14 @@ pub(super) fn collect_tensors(
31
float_dtype: ModelDtype,
32
) -> Result<HashMap<String, KwargValue>> {
33
let PreprocessedImages {
34
- pixel_values,
34
+ encoder_input,
35
model_specific,
36
..
37
} = preprocessed;
38
39
let pixel_values = {
40
- let shape = pixel_values.shape().to_vec();
41
- let data = pixel_values.into_iter().collect();
40
+ let shape = encoder_input.shape().to_vec();
41
+ let data = encoder_input.into_iter().collect();
42
KwargValue::from_f32_tensor(data, shape, float_dtype)?
43
};
44
rust/src/server/src/routes/tests.rs
+1
-1
@@ -559,7 +559,7 @@ fn qwen_multimodal_model_info() -> vllm_chat::multimodal::MultimodalModelInfo {
559
));
560
fs::write(
561
&config_path,
562
- r#"{"model_type":"qwen2_vl","vision_token_id":151655}"#,
562
+ r#"{"model_type":"qwen2_vl","image_token_id":151655}"#,
563
)
564
.expect("write qwen test config");
565
let info = vllm_chat::multimodal::MultimodalModelInfo::from_paths(