@tduncan / Nvidia-Optimization-Stack-VLLM / commits / e281ac663a

[Rust Frontend] Integrate MM audio support (#48554)

Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com>

Bugen Zhao committed Jul 15, 2026 at 15:00 UTC e281ac663a7b772f8b988f270521cd1155f12aa3
18 files changed +1274 -257
rust/Cargo.lock
+403 -11
@@ -182,7 +182,7 @@ dependencies = [
182 "hex",
183 "hmac",
184 "rand 0.9.2",
185 - "reqwest",
185 + "reqwest 0.12.28",
186 "reqwest-eventsource",
187 "secrecy",
188 "serde",
@@ -379,6 +379,12 @@ version = "0.22.1"
379 source = "registry+https://github.com/rust-lang/crates.io-index"
380 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
381
382 +[[package]]
383 +name = "base64ct"
384 +version = "1.8.3"
385 +source = "registry+https://github.com/rust-lang/crates.io-index"
386 +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
387 +
388 [[package]]
389 name = "bit-set"
390 version = "0.5.3"
@@ -665,6 +671,35 @@ version = "0.4.2"
671 source = "registry+https://github.com/rust-lang/crates.io-index"
672 checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b"
673
674 +[[package]]
675 +name = "cookie"
676 +version = "0.18.1"
677 +source = "registry+https://github.com/rust-lang/crates.io-index"
678 +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747"
679 +dependencies = [
680 + "percent-encoding",
681 + "time",
682 + "version_check",
683 +]
684 +
685 +[[package]]
686 +name = "cookie_store"
687 +version = "0.22.1"
688 +source = "registry+https://github.com/rust-lang/crates.io-index"
689 +checksum = "15b2c103cf610ec6cae3da84a766285b42fd16aad564758459e6ecf128c75206"
690 +dependencies = [
691 + "cookie",
692 + "document-features",
693 + "idna",
694 + "indexmap 2.13.0",
695 + "log",
696 + "serde",
697 + "serde_derive",
698 + "serde_json",
699 + "time",
700 + "url",
701 +]
702 +
703 [[package]]
704 name = "core-foundation"
705 version = "0.9.4"
@@ -888,6 +923,16 @@ dependencies = [
923 "serde",
924 ]
925
926 +[[package]]
927 +name = "der"
928 +version = "0.8.1"
929 +source = "registry+https://github.com/rust-lang/crates.io-index"
930 +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d"
931 +dependencies = [
932 + "pem-rfc7468",
933 + "zeroize",
934 +]
935 +
936 [[package]]
937 name = "deranged"
938 version = "0.5.8"
@@ -1171,6 +1216,12 @@ dependencies = [
1216 "once_cell",
1217 ]
1218
1219 +[[package]]
1220 +name = "extended"
1221 +version = "0.1.0"
1222 +source = "registry+https://github.com/rust-lang/crates.io-index"
1223 +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
1224 +
1225 [[package]]
1226 name = "fancy-regex"
1227 version = "0.13.0"
@@ -1589,13 +1640,15 @@ dependencies = [
1640 "indicatif",
1641 "libc",
1642 "log",
1643 + "native-tls",
1644 "num_cpus",
1645 "rand 0.9.2",
1594 - "reqwest",
1646 + "reqwest 0.12.28",
1647 "serde",
1648 "serde_json",
1649 "thiserror 2.0.18",
1650 "tokio",
1651 + "ureq",
1652 "windows-sys 0.61.2",
1653 ]
1654
@@ -2167,22 +2220,27 @@ checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092"
2220 [[package]]
2221 name = "llm-multimodal"
2222 version = "1.7.1"
2170 -source = "git+https://github.com/smg-project/llm-multimodal?rev=c8a29dcc755139fdc26185f400ea48c6d6d48273#c8a29dcc755139fdc26185f400ea48c6d6d48273"
2223 +source = "git+https://github.com/smg-project/llm-multimodal?rev=7df38e53f99aefaebe86934f010aa8084ec99b2f#7df38e53f99aefaebe86934f010aa8084ec99b2f"
2224 dependencies = [
2225 "anyhow",
2226 "base64 0.22.1",
2227 "blake3",
2228 "bytes",
2229 + "cc",
2230 "fast_image_resize",
2231 "hf-hub",
2232 "image",
2233 "libloading",
2234 "ndarray 0.17.2",
2235 "once_cell",
2182 - "reqwest",
2236 + "pkg-config",
2237 + "rayon",
2238 + "reqwest 0.13.4",
2239 + "rustfft",
2240 "serde",
2241 "serde_json",
2242 "serde_with",
2243 + "symphonia",
2244 "tempfile",
2245 "thiserror 2.0.18",
2246 "tokio",
@@ -2740,6 +2798,15 @@ dependencies = [
2798 "pkg-config",
2799 ]
2800
2801 +[[package]]
2802 +name = "pem-rfc7468"
2803 +version = "1.0.0"
2804 +source = "registry+https://github.com/rust-lang/crates.io-index"
2805 +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9"
2806 +dependencies = [
2807 + "base64ct",
2808 +]
2809 +
2810 [[package]]
2811 name = "percent-encoding"
2812 version = "2.3.2"
@@ -3245,9 +3312,9 @@ checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
3312
3313 [[package]]
3314 name = "rayon"
3248 -version = "1.11.0"
3315 +version = "1.12.0"
3316 source = "registry+https://github.com/rust-lang/crates.io-index"
3250 -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
3317 +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d"
3318 dependencies = [
3319 "either",
3320 "rayon-core",
@@ -3346,6 +3413,12 @@ dependencies = [
3413 "regex-syntax",
3414 ]
3415
3416 +[[package]]
3417 +name = "regex-lite"
3418 +version = "0.1.9"
3419 +source = "registry+https://github.com/rust-lang/crates.io-index"
3420 +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
3421 +
3422 [[package]]
3423 name = "regex-syntax"
3424 version = "0.8.10"
@@ -3392,7 +3465,43 @@ dependencies = [
3465 "url",
3466 "wasm-bindgen",
3467 "wasm-bindgen-futures",
3395 - "wasm-streams",
3468 + "wasm-streams 0.4.2",
3469 + "web-sys",
3470 +]
3471 +
3472 +[[package]]
3473 +name = "reqwest"
3474 +version = "0.13.4"
3475 +source = "registry+https://github.com/rust-lang/crates.io-index"
3476 +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
3477 +dependencies = [
3478 + "base64 0.22.1",
3479 + "bytes",
3480 + "futures-core",
3481 + "futures-util",
3482 + "http",
3483 + "http-body",
3484 + "http-body-util",
3485 + "hyper",
3486 + "hyper-tls",
3487 + "hyper-util",
3488 + "js-sys",
3489 + "log",
3490 + "native-tls",
3491 + "percent-encoding",
3492 + "pin-project-lite",
3493 + "rustls-pki-types",
3494 + "sync_wrapper",
3495 + "tokio",
3496 + "tokio-native-tls",
3497 + "tokio-util",
3498 + "tower",
3499 + "tower-http",
3500 + "tower-service",
3501 + "url",
3502 + "wasm-bindgen",
3503 + "wasm-bindgen-futures",
3504 + "wasm-streams 0.5.0",
3505 "web-sys",
3506 ]
3507
@@ -3408,7 +3517,7 @@ dependencies = [
3517 "mime",
3518 "nom",
3519 "pin-project-lite",
3411 - "reqwest",
3520 + "reqwest 0.12.28",
3521 "thiserror 1.0.69",
3522 ]
3523
@@ -3526,7 +3635,9 @@ version = "0.23.37"
3635 source = "registry+https://github.com/rust-lang/crates.io-index"
3636 checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4"
3637 dependencies = [
3638 + "log",
3639 "once_cell",
3640 + "ring",
3641 "rustls-pki-types",
3642 "rustls-webpki",
3643 "subtle",
@@ -4052,6 +4163,17 @@ dependencies = [
4163 "windows-sys 0.61.2",
4164 ]
4165
4166 +[[package]]
4167 +name = "socks"
4168 +version = "0.3.4"
4169 +source = "registry+https://github.com/rust-lang/crates.io-index"
4170 +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
4171 +dependencies = [
4172 + "byteorder",
4173 + "libc",
4174 + "winapi",
4175 +]
4176 +
4177 [[package]]
4178 name = "spm_precompiled"
4179 version = "0.1.4"
@@ -4127,6 +4249,203 @@ version = "2.6.1"
4249 source = "registry+https://github.com/rust-lang/crates.io-index"
4250 checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
4251
4252 +[[package]]
4253 +name = "symphonia"
4254 +version = "0.6.0"
4255 +source = "registry+https://github.com/rust-lang/crates.io-index"
4256 +checksum = "1758d6c853020a7244de03cc3e0185eaea3f58715122422dd3cc7452e6d4c16a"
4257 +dependencies = [
4258 + "lazy_static",
4259 + "symphonia-bundle-flac",
4260 + "symphonia-bundle-mp3",
4261 + "symphonia-codec-aac",
4262 + "symphonia-codec-adpcm",
4263 + "symphonia-codec-alac",
4264 + "symphonia-codec-pcm",
4265 + "symphonia-codec-vorbis",
4266 + "symphonia-core",
4267 + "symphonia-format-caf",
4268 + "symphonia-format-isomp4",
4269 + "symphonia-format-mkv",
4270 + "symphonia-format-ogg",
4271 + "symphonia-format-riff",
4272 + "symphonia-metadata",
4273 +]
4274 +
4275 +[[package]]
4276 +name = "symphonia-bundle-flac"
4277 +version = "0.6.0"
4278 +source = "registry+https://github.com/rust-lang/crates.io-index"
4279 +checksum = "ee69ad01236a67260b82fd1ff9790dd75ead29f2f46af145e63b7e72273e0e03"
4280 +dependencies = [
4281 + "log",
4282 + "symphonia-common",
4283 + "symphonia-core",
4284 + "symphonia-metadata",
4285 +]
4286 +
4287 +[[package]]
4288 +name = "symphonia-bundle-mp3"
4289 +version = "0.6.0"
4290 +source = "registry+https://github.com/rust-lang/crates.io-index"
4291 +checksum = "350f1f2f2e19ad4dd315db94304d1eb361b29af070681f94e51b8fdaad769546"
4292 +dependencies = [
4293 + "lazy_static",
4294 + "log",
4295 + "symphonia-core",
4296 +]
4297 +
4298 +[[package]]
4299 +name = "symphonia-codec-aac"
4300 +version = "0.6.0"
4301 +source = "registry+https://github.com/rust-lang/crates.io-index"
4302 +checksum = "f1979c515a76371b186aad2feff5f23e21cbec775bf95de08bf1e3af92a2ad76"
4303 +dependencies = [
4304 + "lazy_static",
4305 + "log",
4306 + "symphonia-common",
4307 + "symphonia-core",
4308 +]
4309 +
4310 +[[package]]
4311 +name = "symphonia-codec-adpcm"
4312 +version = "0.6.0"
4313 +source = "registry+https://github.com/rust-lang/crates.io-index"
4314 +checksum = "4ebbdfd76d6cc5a601c6292a44357c5b7c82f2cd7cdc0f171421f5c5cff0ea1f"
4315 +dependencies = [
4316 + "log",
4317 + "symphonia-core",
4318 +]
4319 +
4320 +[[package]]
4321 +name = "symphonia-codec-alac"
4322 +version = "0.6.0"
4323 +source = "registry+https://github.com/rust-lang/crates.io-index"
4324 +checksum = "3a149cbfc7fb5c405d123a273227d31de17138419552112bf1aa7b73e65827b8"
4325 +dependencies = [
4326 + "log",
4327 + "symphonia-common",
4328 + "symphonia-core",
4329 +]
4330 +
4331 +[[package]]
4332 +name = "symphonia-codec-pcm"
4333 +version = "0.6.0"
4334 +source = "registry+https://github.com/rust-lang/crates.io-index"
4335 +checksum = "50baee168f0e9dcf6ba7fc06e8b57eb62072a4490cc7cf13af77e72baae5d328"
4336 +dependencies = [
4337 + "log",
4338 + "symphonia-core",
4339 +]
4340 +
4341 +[[package]]
4342 +name = "symphonia-codec-vorbis"
4343 +version = "0.6.0"
4344 +source = "registry+https://github.com/rust-lang/crates.io-index"
4345 +checksum = "45b07b4423cd8e0fc472575909a5554b12c2f58e3c190b38c24f042e732fd8de"
4346 +dependencies = [
4347 + "log",
4348 + "symphonia-common",
4349 + "symphonia-core",
4350 +]
4351 +
4352 +[[package]]
4353 +name = "symphonia-common"
4354 +version = "0.6.0"
4355 +source = "registry+https://github.com/rust-lang/crates.io-index"
4356 +checksum = "8257891ffa7f05e02b58f4761e2abf7e5278c8744fd59e981559e050f86eef55"
4357 +dependencies = [
4358 + "log",
4359 + "symphonia-core",
4360 + "symphonia-metadata",
4361 +]
4362 +
4363 +[[package]]
4364 +name = "symphonia-core"
4365 +version = "0.6.0"
4366 +source = "registry+https://github.com/rust-lang/crates.io-index"
4367 +checksum = "95ec293b5f288383b72a7bffcade6b2860b642cf66f28b3bd5967349a49938b1"
4368 +dependencies = [
4369 + "bitflags",
4370 + "bytemuck",
4371 + "lazy_static",
4372 + "log",
4373 + "num-complex",
4374 + "smallvec",
4375 +]
4376 +
4377 +[[package]]
4378 +name = "symphonia-format-caf"
4379 +version = "0.6.0"
4380 +source = "registry+https://github.com/rust-lang/crates.io-index"
4381 +checksum = "cde3ca76633d3400ab57195456c09f8a58d775ff5452329f3f212b6efc8622f5"
4382 +dependencies = [
4383 + "log",
4384 + "symphonia-common",
4385 + "symphonia-core",
4386 +]
4387 +
4388 +[[package]]
4389 +name = "symphonia-format-isomp4"
4390 +version = "0.6.0"
4391 +source = "registry+https://github.com/rust-lang/crates.io-index"
4392 +checksum = "2d179a01305b3505940135a9f0180d6ef4b487912748fe97554756f120fbd05e"
4393 +dependencies = [
4394 + "log",
4395 + "symphonia-common",
4396 + "symphonia-core",
4397 + "symphonia-metadata",
4398 +]
4399 +
4400 +[[package]]
4401 +name = "symphonia-format-mkv"
4402 +version = "0.6.0"
4403 +source = "registry+https://github.com/rust-lang/crates.io-index"
4404 +checksum = "fb17713e134f5ad316c2690fa3104590ccc85842cdbcf82c3cd1a845cb08aa74"
4405 +dependencies = [
4406 + "lazy_static",
4407 + "log",
4408 + "symphonia-common",
4409 + "symphonia-core",
4410 +]
4411 +
4412 +[[package]]
4413 +name = "symphonia-format-ogg"
4414 +version = "0.6.0"
4415 +source = "registry+https://github.com/rust-lang/crates.io-index"
4416 +checksum = "b05a67e02b1e4fca1a261ba4fe06910a9357489ad8c36aafdd2960e9c6559433"
4417 +dependencies = [
4418 + "log",
4419 + "symphonia-common",
4420 + "symphonia-core",
4421 + "symphonia-metadata",
4422 +]
4423 +
4424 +[[package]]
4425 +name = "symphonia-format-riff"
4426 +version = "0.6.0"
4427 +source = "registry+https://github.com/rust-lang/crates.io-index"
4428 +checksum = "17424452a777666d3eaf09a5c651029b15b6a333812fcc5b5474f2a3f0cff3f0"
4429 +dependencies = [
4430 + "extended",
4431 + "log",
4432 + "symphonia-core",
4433 + "symphonia-metadata",
4434 +]
4435 +
4436 +[[package]]
4437 +name = "symphonia-metadata"
4438 +version = "0.6.0"
4439 +source = "registry+https://github.com/rust-lang/crates.io-index"
4440 +checksum = "a31acf5cd623398a6208e2225d18f4b20f761c55098a796a5247ad516a4a8681"
4441 +dependencies = [
4442 + "lazy_static",
4443 + "log",
4444 + "regex-lite",
4445 + "smallvec",
4446 + "symphonia-core",
4447 +]
4448 +
4449 [[package]]
4450 name = "syn"
4451 version = "1.0.109"
@@ -4982,6 +5301,42 @@ version = "0.9.0"
5301 source = "registry+https://github.com/rust-lang/crates.io-index"
5302 checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
5303
5304 +[[package]]
5305 +name = "ureq"
5306 +version = "3.3.0"
5307 +source = "registry+https://github.com/rust-lang/crates.io-index"
5308 +checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0"
5309 +dependencies = [
5310 + "base64 0.22.1",
5311 + "cookie_store",
5312 + "der",
5313 + "flate2",
5314 + "log",
5315 + "native-tls",
5316 + "percent-encoding",
5317 + "rustls",
5318 + "rustls-pki-types",
5319 + "serde",
5320 + "serde_json",
5321 + "socks",
5322 + "ureq-proto",
5323 + "utf8-zero",
5324 + "webpki-root-certs",
5325 + "webpki-roots",
5326 +]
5327 +
5328 +[[package]]
5329 +name = "ureq-proto"
5330 +version = "0.6.0"
5331 +source = "registry+https://github.com/rust-lang/crates.io-index"
5332 +checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c"
5333 +dependencies = [
5334 + "base64 0.22.1",
5335 + "http",
5336 + "httparse",
5337 + "log",
5338 +]
5339 +
5340 [[package]]
5341 name = "url"
5342 version = "2.5.8"
@@ -5006,6 +5361,12 @@ version = "1.0.5"
5361 source = "registry+https://github.com/rust-lang/crates.io-index"
5362 checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246"
5363
5364 +[[package]]
5365 +name = "utf8-zero"
5366 +version = "0.8.1"
5367 +source = "registry+https://github.com/rust-lang/crates.io-index"
5368 +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
5369 +
5370 [[package]]
5371 name = "utf8_iter"
5372 version = "1.0.4"
@@ -5097,7 +5458,7 @@ dependencies = [
5458 "ndarray 0.17.2",
5459 "oss-harmony",
5460 "paste",
5100 - "reqwest",
5461 + "reqwest 0.13.4",
5462 "rmp-serde",
5463 "serde",
5464 "serde-json-fmt",
@@ -5344,7 +5705,7 @@ dependencies = [
5705 "futures",
5706 "hf-hub",
5707 "itertools 0.14.0",
5347 - "reqwest",
5708 + "reqwest 0.12.28",
5709 "serde",
5710 "serde_json",
5711 "serde_with",
@@ -5368,7 +5729,7 @@ dependencies = [
5729 "criterion",
5730 "fastokens",
5731 "hf-hub",
5371 - "reqwest",
5732 + "reqwest 0.12.28",
5733 "riptoken",
5734 "rustc-hash 1.1.0",
5735 "serde",
@@ -5531,6 +5892,19 @@ dependencies = [
5892 "web-sys",
5893 ]
5894
5895 +[[package]]
5896 +name = "wasm-streams"
5897 +version = "0.5.0"
5898 +source = "registry+https://github.com/rust-lang/crates.io-index"
5899 +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb"
5900 +dependencies = [
5901 + "futures-util",
5902 + "js-sys",
5903 + "wasm-bindgen",
5904 + "wasm-bindgen-futures",
5905 + "web-sys",
5906 +]
5907 +
5908 [[package]]
5909 name = "wasmparser"
5910 version = "0.244.0"
@@ -5563,6 +5937,24 @@ dependencies = [
5937 "wasm-bindgen",
5938 ]
5939
5940 +[[package]]
5941 +name = "webpki-root-certs"
5942 +version = "1.0.8"
5943 +source = "registry+https://github.com/rust-lang/crates.io-index"
5944 +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267"
5945 +dependencies = [
5946 + "rustls-pki-types",
5947 +]
5948 +
5949 +[[package]]
5950 +name = "webpki-roots"
5951 +version = "1.0.8"
5952 +source = "registry+https://github.com/rust-lang/crates.io-index"
5953 +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf"
5954 +dependencies = [
5955 + "rustls-pki-types",
5956 +]
5957 +
5958 [[package]]
5959 name = "weezl"
5960 version = "0.1.12"
rust/Cargo.toml
+2 -1
@@ -53,7 +53,7 @@ hyper-util = { version = "0.1.20", features = [
53 indexmap = "2.13.0"
54 itertools = "0.14.0"
55 libc = "0.2.177"
56 -llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "c8a29dcc755139fdc26185f400ea48c6d6d48273" }
56 +llm-multimodal = { git = "https://github.com/smg-project/llm-multimodal", rev = "7df38e53f99aefaebe86934f010aa8084ec99b2f", default-features = false, features = ["native-tls"] }
57 mimalloc = "0.1.52"
58 minijinja = { version = "2.0", features = ["unstable_machinery", "json", "builtins", "loader", "loop_controls", "preserve_order"] }
59 minijinja-contrib = { version = "2.0", features = ["pycompat"] }
@@ -73,6 +73,7 @@ pythonize = "0.28.0"
73 rand = "0.9.2"
74 reasoning-parser = "1.2.2"
75 reqwest = { version = "0.12.8", default-features = false, features = ["native-tls"] }
76 +reqwest-0-13 = { package = "reqwest", version = "0.13.4", default-features = false, features = ["native-tls"] }
77 riptoken = { version = "0.3.0", default-features = false }
78 rmp-serde = "1.3.1"
79 rmpv = { version = "1.3.1", features = ["with-serde"] }
rust/src/chat/Cargo.toml
+1 -1
@@ -16,7 +16,7 @@ llm-multimodal.workspace = true
16 minijinja.workspace = true
17 minijinja-contrib.workspace = true
18 openai-harmony.workspace = true
19 -reqwest.workspace = true
19 +reqwest-0-13.workspace = true
20 serde.workspace = true
21 serde-json-fmt.workspace = true
22 serde_json.workspace = true
rust/src/chat/src/backend/hf.rs
+1
@@ -151,6 +151,7 @@ fn resolve_multimodal_render_info(
151 info.map(|info| MultimodalRenderInfo {
152 image_token: info.placeholder_token(Modality::Image).map(str::to_string),
153 video_token: info.placeholder_token(Modality::Video).map(str::to_string),
154 + audio_token: info.placeholder_token(Modality::Audio).map(str::to_string),
155 })
156 }
157
rust/src/chat/src/multimodal.rs
+202 -91
@@ -5,24 +5,24 @@
5 //!
6 //! This module owns the multimodal path for chat requests: it extracts media
7 //! parts from structured chat messages, fetches and preprocesses them through
8 -//! `llm-multimodal` one modality at a time, expands rendered prompt
9 -//! placeholders after tokenization, and builds the engine-facing
10 -//! `MmFeatures` payload.
8 +//! `llm-multimodal`, expands rendered prompt placeholders after tokenization,
9 +//! and builds the engine-facing `MmFeatures` payload.
10 //!
11 //! Raw media stays above `vllm-text`; this module lowers it into token IDs and
12 //! opaque tensor payloads before the request is handed to text generation.
13
15 -use std::collections::{HashMap, HashSet};
14 +use std::collections::HashSet;
15 use std::fs;
16 use std::path::Path;
17 use std::sync::{Arc, LazyLock};
18
19 use itertools::izip;
20 use llm_multimodal::{
22 - AsyncMultiModalTracker, FieldLayout, ImageFrame, MediaConnector, MediaConnectorConfig,
23 - MediaContentPart, Modality, ModelMetadata, ModelProcessorSpec, ModelRegistry,
24 - PreProcessorConfig, PreprocessedEncoderInputs, PromptReplacement, Tokenizer as TokenResolver,
25 - TrackedMedia, VideoClip, VisionPreProcessor, VisionProcessorRegistry,
21 + AsyncMultiModalTracker, AudioClip, AudioPreProcessor, EncoderFieldLayouts, FieldLayout,
22 + ImageFrame, MediaConnector, MediaConnectorConfig, MediaContentPart, Modality, ModelMetadata,
23 + ModelProcessorSpec, ModelRegistry, PreProcessorConfig, PreprocessedEncoderInputs,
24 + PromptReplacement, Tokenizer as TokenResolver, TrackedMedia, VideoClip, VisionPreProcessor,
25 + VisionProcessorRegistry,
26 };
27 use thiserror_ext::AsReport as _;
28 use tracing::warn;
@@ -35,8 +35,10 @@ use crate::error::{Error, Result, bail_multimodal, multimodal};
35 use crate::renderer::RenderedPrompt;
36 use crate::request::{ChatContent, ChatContentPart, ChatMessage, ChatRequest};
37
38 +mod audio;
39 mod expand;
40 mod image;
41 +mod item;
42 mod tensor;
43 mod video;
44
@@ -46,9 +48,9 @@ use self::expand::expand_prompt_token_ids;
48 #[derive(Clone)]
49 pub struct MultimodalModelInfo {
50 context: MultimodalModelContext,
49 - spec: ResolvedMultimodalSpec,
51 image: Option<ModalitySupport>,
52 video: Option<ModalitySupport>,
53 + audio: Option<AudioModalitySupport>,
54 media_connector: Arc<MediaConnector>,
55 }
56
@@ -88,22 +90,50 @@ impl MultimodalModelContext {
90 LazyLock::new(VisionProcessorRegistry::with_defaults);
91 REGISTRY.find(&self.model_id, self.model_type.as_deref())
92 }
93 +
94 + /// Resolve an audio preprocessor for one loaded model.
95 + fn resolve_audio_processor(
96 + &self,
97 + model_spec: &'static dyn ModelProcessorSpec,
98 + preprocessor_config: &PreProcessorConfig,
99 + ) -> Option<Arc<dyn AudioPreProcessor>> {
100 + model_spec.audio_processor(&self.config, preprocessor_config).map(Arc::from)
101 + }
102 }
103
93 -/// Static model-specific tensor-layout behavior shared across modalities.
104 +/// Static model-specific tensor-layout behavior for one modality.
105 #[derive(Clone)]
106 struct ResolvedMultimodalSpec {
107 raw: &'static dyn ModelProcessorSpec,
97 - field_layouts: HashMap<String, FieldLayout>,
108 + modality: Modality,
109 + field_layouts: EncoderFieldLayouts,
110 keep_on_cpu_keys: HashSet<String>,
111 }
112
113 impl ResolvedMultimodalSpec {
102 - fn new(raw: &'static dyn ModelProcessorSpec) -> Self {
114 + fn new(raw: &'static dyn ModelProcessorSpec, modality: Modality) -> Self {
115 Self {
116 raw,
105 - field_layouts: raw.field_layouts(),
106 - keep_on_cpu_keys: raw.keep_on_cpu_keys().into_iter().collect(),
117 + modality,
118 + field_layouts: raw.encoder_field_layouts_for(modality),
119 + keep_on_cpu_keys: raw.keep_on_cpu_keys_for(modality).into_iter().collect(),
120 + }
121 + }
122 +
123 + fn primary_key(&self) -> &'static str {
124 + match self.modality {
125 + Modality::Image => image::IMAGE_PRIMARY_KEY,
126 + Modality::Video => video::VIDEO_PRIMARY_KEY,
127 + Modality::Audio => audio::AUDIO_PRIMARY_KEY,
128 + Modality::ImageEmbeds => unreachable!("image embeds use no encoder preprocessor"),
129 + }
130 + }
131 +
132 + fn field_layout_for(&self, key: &str) -> Option<&FieldLayout> {
133 + if key == self.primary_key() {
134 + Some(&self.field_layouts.encoder_input)
135 + } else {
136 + self.field_layouts.model_specific.get(key)
137 }
138 }
139
@@ -111,14 +141,15 @@ impl ResolvedMultimodalSpec {
141 &self,
142 context: &MultimodalModelContext,
143 preprocessed: &PreprocessedEncoderInputs,
114 - modality: Modality,
144 ) -> Result<Vec<PromptReplacement>> {
116 - Ok(self.raw.prompt_replacements_for(&context.metadata(), preprocessed, modality)?)
145 + Ok(self
146 + .raw
147 + .prompt_replacements_for(&context.metadata(), preprocessed, self.modality)?)
148 }
149 }
150
151 /// Resolved placeholder tokens for one modality.
121 -#[derive(Clone)]
152 +#[derive(Debug, Clone)]
153 struct ResolvedPlaceholder {
154 token: String,
155 /// The token ID emitted for `token` in the rendered prompt.
@@ -152,19 +183,30 @@ impl ResolvedPlaceholder {
183 }
184 }
185
155 -/// Static per-modality vision preprocessor plus its loaded config and
156 -/// resolved placeholder tokens.
186 +/// Static per-modality vision preprocessor plus its loaded config, resolved
187 +/// placeholder tokens, and the model's shared tensor-layout spec.
188 #[derive(Clone)]
189 struct ModalitySupport {
190 + spec: ResolvedMultimodalSpec,
191 placeholder: ResolvedPlaceholder,
192 processor: &'static dyn VisionPreProcessor,
193 config: PreProcessorConfig,
194 }
195
196 +/// Static audio preprocessor plus the resolved model contract for its output.
197 +#[derive(Clone)]
198 +struct AudioModalitySupport {
199 + spec: ResolvedMultimodalSpec,
200 + placeholder: ResolvedPlaceholder,
201 + processor: Arc<dyn AudioPreProcessor>,
202 +}
203 +
204 /// Model-repo config file locations consumed by multimodal support.
205 #[derive(Debug, Default, Clone, Copy)]
206 pub struct MultimodalConfigFiles<'a> {
207 pub config: Option<&'a Path>,
208 + /// Shared preprocessor config (`preprocessor_config.json`), consumed by
209 + /// image/audio processors and used as the video fallback.
210 pub preprocessor_config: Option<&'a Path>,
211 /// Video-specific preprocessor config (`video_preprocessor_config.json`).
212 pub video_preprocessor_config: Option<&'a Path>,
@@ -212,6 +254,8 @@ struct FetchedMedia {
254 image_uuids: Vec<Option<String>>,
255 videos: Vec<Arc<VideoClip>>,
256 video_uuids: Vec<Option<String>>,
257 + audios: Vec<Arc<AudioClip>>,
258 + audio_uuids: Vec<Option<String>>,
259 }
260
261 /// One modality's preprocessed output, ready for the shared expansion and
@@ -253,7 +297,7 @@ impl MultimodalModelInfo {
297 }
298 None => serde_json::Value::Object(Default::default()),
299 };
256 - let image_preprocessor_config = load_preprocessor_config(
300 + let preprocessor_config = load_preprocessor_config(
301 files.preprocessor_config,
302 "preprocessor_config.json",
303 files.processor_config,
@@ -266,7 +310,7 @@ impl MultimodalModelInfo {
310 files.processor_config,
311 "video_processor",
312 )?
269 - .unwrap_or_else(|| image_preprocessor_config.clone());
313 + .unwrap_or_else(|| preprocessor_config.clone());
314
315 let context = MultimodalModelContext {
316 model_id,
@@ -275,41 +319,76 @@ impl MultimodalModelInfo {
319 tokenizer: TokenizerResolver(tokenizer),
320 };
321
278 - Self::from_loaded(
279 - context,
280 - image_preprocessor_config,
281 - video_preprocessor_config,
282 - )
322 + Self::from_loaded(context, preprocessor_config, video_preprocessor_config)
323 }
324
325 /// Resolve multimodal support from an assembled context and parsed
326 /// preprocessor configs.
327 fn from_loaded(
328 context: MultimodalModelContext,
289 - image_preprocessor_config: PreProcessorConfig,
329 + preprocessor_config: PreProcessorConfig,
330 video_preprocessor_config: PreProcessorConfig,
331 ) -> Result<Option<Self>> {
292 - let Some(raw_spec) = context.resolve_model_spec() else {
332 + let (image, video) = Self::resolve_vision_lanes(
333 + &context,
334 + preprocessor_config.clone(),
335 + video_preprocessor_config,
336 + );
337 + let audio = Self::resolve_audio_lane(&context, &preprocessor_config)?;
338 +
339 + if image.is_none() && video.is_none() && audio.is_none() {
340 warn!(
341 model_id = context.model_id,
342 model_type = context.model_type,
296 - "multimodal model spec is not registered; disabling multimodal support for this model"
343 + "no multimodal modality resolved; disabling multimodal support for this model"
344 );
345 return Ok(None);
346 + }
347 +
348 + let media_connector = Arc::new(MediaConnector::new(
349 + reqwest_0_13::Client::new(),
350 + MediaConnectorConfig::default(),
351 + )?);
352 +
353 + Ok(Some(Self {
354 + context,
355 + image,
356 + video,
357 + audio,
358 + media_connector,
359 + }))
360 + }
361 +
362 + /// Resolve the spec-backed vision lanes (image and video).
363 + ///
364 + /// A missing model spec or vision processor disables both lanes but is
365 + /// not an error: unsupported model families are logged and skipped.
366 + fn resolve_vision_lanes(
367 + context: &MultimodalModelContext,
368 + preprocessor_config: PreProcessorConfig,
369 + video_preprocessor_config: PreProcessorConfig,
370 + ) -> (Option<ModalitySupport>, Option<ModalitySupport>) {
371 + let Some(raw_spec) = context.resolve_model_spec() else {
372 + warn!(
373 + model_id = context.model_id,
374 + model_type = context.model_type,
375 + "multimodal model spec is not registered; disabling image/video support for this model"
376 + );
377 + return (None, None);
378 };
379
380 let Some(processor) = context.resolve_vision_processor() else {
381 warn!(
382 model_id = context.model_id,
383 model_type = context.model_type,
305 - "vision processor is not registered; disabling multimodal support for this model"
384 + "vision processor is not registered; disabling image/video support for this model"
385 );
307 - return Ok(None);
386 + return (None, None);
387 };
388
389 // Warn and disable the modality if the placeholder resolution fails.
390 let resolve_placeholder =
312 - |modality: Modality| match ResolvedPlaceholder::resolve(raw_spec, &context, modality) {
391 + |modality: Modality| match ResolvedPlaceholder::resolve(raw_spec, context, modality) {
392 Ok(placeholder) => Some(placeholder),
393 Err(error) => {
394 warn!(
@@ -323,9 +402,10 @@ impl MultimodalModelInfo {
402 };
403
404 let image = resolve_placeholder(Modality::Image).map(|placeholder| ModalitySupport {
405 + spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Image),
406 placeholder,
407 processor,
328 - config: image_preprocessor_config,
408 + config: preprocessor_config.clone(),
409 });
410
411 let video = resolve_placeholder(Modality::Video).and_then(|placeholder| {
@@ -341,6 +421,7 @@ impl MultimodalModelInfo {
421 None
422 } else {
423 Some(ModalitySupport {
424 + spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Video),
425 placeholder,
426 processor,
427 config: video_preprocessor_config,
@@ -348,26 +429,33 @@ impl MultimodalModelInfo {
429 }
430 });
431
351 - if image.is_none() && video.is_none() {
432 + (image, video)
433 + }
434 +
435 + /// Resolve a spec-backed audio lane when the model advertises audio support.
436 + fn resolve_audio_lane(
437 + context: &MultimodalModelContext,
438 + preprocessor_config: &PreProcessorConfig,
439 + ) -> Result<Option<AudioModalitySupport>> {
440 + let Some(raw_spec) = context.resolve_model_spec() else {
441 + return Ok(None);
442 + };
443 + if !raw_spec.modality_limits(&context.metadata())?.contains_key(&Modality::Audio) {
444 + return Ok(None);
445 + }
446 +
447 + let Some(processor) = context.resolve_audio_processor(raw_spec, preprocessor_config) else {
448 warn!(
449 model_id = context.model_id,
354 - model_type = context.model_type,
355 - "no multimodal modality resolved; disabling multimodal support for this model"
450 + model_spec = raw_spec.name(),
451 + "model spec does not provide an audio processor; disabling audio support for this model"
452 );
453 return Ok(None);
358 - }
359 -
360 - let media_connector = Arc::new(MediaConnector::new(
361 - reqwest::Client::new(),
362 - MediaConnectorConfig::default(),
363 - )?);
364 -
365 - Ok(Some(Self {
366 - context,
367 - spec: ResolvedMultimodalSpec::new(raw_spec),
368 - image,
369 - video,
370 - media_connector,
454 + };
455 + Ok(Some(AudioModalitySupport {
456 + spec: ResolvedMultimodalSpec::new(raw_spec, Modality::Audio),
457 + placeholder: ResolvedPlaceholder::resolve(raw_spec, context, Modality::Audio)?,
458 + processor,
459 }))
460 }
461
@@ -380,7 +468,8 @@ impl MultimodalModelInfo {
468 match modality {
469 Modality::Image => self.image.as_ref()?.placeholder.token.as_str().into(),
470 Modality::Video => self.video.as_ref()?.placeholder.token.as_str().into(),
383 - _ => None,
471 + Modality::Audio => self.audio.as_ref()?.placeholder.token.as_str().into(),
472 + Modality::ImageEmbeds => None,
473 }
474 }
475 }
@@ -449,12 +538,35 @@ fn extract_media_parts(request: &ChatRequest) -> Result<Vec<MediaContentPart>> {
538 uuid: uuid.clone(),
539 })
540 }
541 + ChatContentPart::InputAudio { data, format, uuid } => {
542 + all_parts.push(MediaContentPart::AudioUrl {
543 + url: input_audio_data_url(data, format.as_deref())?,
544 + uuid: uuid.clone(),
545 + })
546 + }
547 + ChatContentPart::AudioUrl { audio_url, uuid } => {
548 + all_parts.push(MediaContentPart::AudioUrl {
549 + url: audio_url.clone(),
550 + uuid: uuid.clone(),
551 + })
552 + }
553 }
554 }
555 }
556 Ok(all_parts)
557 }
558
559 +/// Wrap OpenAI base64 audio in a data URL consumed by `MediaConnector`.
560 +fn input_audio_data_url(data: &str, format: Option<&str>) -> Result<String> {
561 + let mime_type = match format {
562 + Some(format) if format.eq_ignore_ascii_case("wav") => "audio/wav",
563 + Some(format) if format.eq_ignore_ascii_case("mp3") => "audio/mpeg",
564 + Some(format) => bail_multimodal!("unsupported input_audio format `{format}`"),
565 + None => "application/octet-stream",
566 + };
567 + Ok(format!("data:{mime_type};base64,{data}"))
568 +}
569 +
570 impl MultimodalModelInfo {
571 /// Run media fetch, per-modality preprocessing, prompt expansion, and
572 /// feature build.
@@ -468,11 +580,10 @@ impl MultimodalModelInfo {
580 prompt_token_ids: &mut Vec<u32>,
581 model_dtype: ModelDtype,
582 ) -> Result<MmFeatures> {
471 - if media_parts.is_empty() {
583 + let media_parts_len = media_parts.len();
584 + if media_parts_len == 0 {
585 return Ok(Vec::new());
586 }
474 - let media_parts_len = media_parts.len();
475 -
587 // TODO: enforce per-modality item-count limits, aligned with the
588 // engine's `--limit-mm-per-prompt` semantics.
589 let fetched = self.fetch_media(media_parts).await?;
@@ -486,6 +597,9 @@ impl MultimodalModelInfo {
597 prepared
598 .push(self.prepare_videos(fetched.videos, fetched.video_uuids, model_dtype).await?);
599 }
600 + if !fetched.audios.is_empty() {
601 + prepared.push(self.prepare_audios(fetched.audios, fetched.audio_uuids).await?);
602 + }
603
604 let mut ranges = expand_prompt_token_ids(prompt_token_ids, &prepared)?;
605
@@ -524,8 +638,8 @@ impl MultimodalModelInfo {
638 Ok(features)
639 }
640
527 - /// Fetch all media parts and split them per modality, preserving their
528 - /// request-order UUID metadata.
641 + /// Fetch all connector-backed media parts and split them per modality,
642 + /// preserving their request-order UUID metadata.
643 async fn fetch_media(&self, media_parts: Vec<MediaContentPart>) -> Result<FetchedMedia> {
644 let mut tracker = AsyncMultiModalTracker::new(Arc::clone(&self.media_connector));
645 for part in media_parts {
@@ -562,11 +676,27 @@ impl MultimodalModelInfo {
676 .collect::<Result<Vec<_>>>()?;
677 let video_uuids = tracker_output.uuids.remove(&Modality::Video).unwrap_or_default();
678
679 + let audios = tracker_output
680 + .data
681 + .remove(&Modality::Audio)
682 + .unwrap_or_default()
683 + .into_iter()
684 + .map(|media| match media {
685 + TrackedMedia::Audio(clip) => Ok(clip),
686 + _ => Err(multimodal!(
687 + "tracker returned non-audio media for the audio modality"
688 + )),
689 + })
690 + .collect::<Result<Vec<_>>>()?;
691 + let audio_uuids = tracker_output.uuids.remove(&Modality::Audio).unwrap_or_default();
692 +
693 Ok(FetchedMedia {
694 images,
695 image_uuids,
696 videos,
697 video_uuids,
698 + audios,
699 + audio_uuids,
700 })
701 }
702 }
@@ -665,42 +795,6 @@ mod tests {
795 test_info("qwen3_vl", config, qwen3_vl_tokenizer())
796 }
797
668 - #[test]
669 - fn from_paths_resolves_image_config_from_processor_config() {
670 - let dir = tempfile::tempdir().unwrap();
671 - let config_path = dir.path().join("config.json");
672 - std::fs::write(
673 - &config_path,
674 - serde_json::json!({
675 - "model_type": "qwen3_vl",
676 - "image_token_id": QWEN3_IMAGE_PAD_ID,
677 - })
678 - .to_string(),
679 - )
680 - .unwrap();
681 - let processor_config_path = dir.path().join("processor_config.json");
682 - std::fs::write(
683 - &processor_config_path,
684 - r#"{"image_processor":{"size":{"shortest_edge":64}}}"#,
685 - )
686 - .unwrap();
687 -
688 - let info = MultimodalModelInfo::from_paths(
689 - "qwen3-vl-test".to_string(),
690 - Some("qwen3_vl".to_string()),
691 - MultimodalConfigFiles {
692 - config: Some(&config_path),
693 - processor_config: Some(&processor_config_path),
694 - ..Default::default()
695 - },
696 - Arc::new(qwen3_vl_tokenizer()),
697 - )
698 - .unwrap()
699 - .unwrap();
700 -
701 - assert_eq!(info.image.unwrap().config.get_shortest_edge(), Some(64));
702 - }
703 -
798 #[test]
799 fn qwen3_vl_resolves_image_and_video_support() {
800 let info = qwen3_vl_info();
@@ -742,4 +836,21 @@ mod tests {
836 assert_eq!(info.placeholder_token(Modality::Image), Some("<|image|>"));
837 assert_eq!(info.placeholder_token(Modality::Video), None);
838 }
839 +
840 + #[test]
841 + fn input_audio_uses_connector_data_urls() {
842 + assert_eq!(
843 + input_audio_data_url("AAAA", Some("wav")).unwrap(),
844 + "data:audio/wav;base64,AAAA"
845 + );
846 + assert_eq!(
847 + input_audio_data_url("BBBB", Some("MP3")).unwrap(),
848 + "data:audio/mpeg;base64,BBBB"
849 + );
850 + assert_eq!(
851 + input_audio_data_url("CCCC", None).unwrap(),
852 + "data:application/octet-stream;base64,CCCC"
853 + );
854 + assert!(input_audio_data_url("AAAA", Some("flac")).is_err());
855 + }
856 }
rust/src/chat/src/multimodal/audio.rs new
+213
@@ -0,0 +1,213 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +//! Audio-modality preparation through `llm-multimodal`.
5 +
6 +use std::sync::Arc;
7 +
8 +use llm_multimodal::{AudioClip, Modality, PreprocessedEncoderInputs};
9 +use vllm_engine_core_client::protocol::dtype::ModelDtype;
10 +
11 +use super::{AudioModalitySupport, MultimodalModelInfo, PreparedMedia, item};
12 +use crate::error::{Error, Result, bail_multimodal, multimodal};
13 +
14 +/// Forward-kwargs name of the primary audio encoder input.
15 +pub(super) const AUDIO_PRIMARY_KEY: &str = "input_audio_features";
16 +
17 +impl MultimodalModelInfo {
18 + /// Preprocess fetched audio clips as one batch and build per-item features.
19 + pub(super) async fn prepare_audios(
20 + &self,
21 + clips: Vec<Arc<AudioClip>>,
22 + uuids: Vec<Option<String>>,
23 + ) -> Result<PreparedMedia> {
24 + let support = self.audio.as_ref().ok_or_else(|| Error::UnsupportedModality {
25 + modality: Modality::Audio.to_string(),
26 + })?;
27 + let preprocessed = self.preprocess_audios(support, &clips).await?;
28 + let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
29 + if replacements.len() != clips.len() {
30 + bail_multimodal!(
31 + "number of audio prompt replacements {} does not match number of audio clips {}",
32 + replacements.len(),
33 + clips.len()
34 + );
35 + }
36 +
37 + let hashes = clips.iter().map(|clip| clip.hash.clone()).collect();
38 + let items = item::build_batched_items(
39 + &support.spec,
40 + preprocessed,
41 + hashes,
42 + uuids,
43 + ModelDtype::Float32,
44 + )?;
45 +
46 + Ok(PreparedMedia {
47 + modality: Modality::Audio,
48 + placeholder: support.placeholder.clone(),
49 + replacements,
50 + items,
51 + })
52 + }
53 +
54 + /// Run CPU-heavy audio preprocessing in a blocking task.
55 + async fn preprocess_audios(
56 + &self,
57 + support: &AudioModalitySupport,
58 + clips: &[Arc<AudioClip>],
59 + ) -> Result<PreprocessedEncoderInputs> {
60 + let processor = Arc::clone(&support.processor);
61 + let clips = clips.to_vec();
62 + tokio::task::spawn_blocking(move || Ok(processor.preprocess(&clips)?))
63 + .await
64 + .map_err(|error| multimodal!("audio preprocessing task failed: {error}"))?
65 + }
66 +}
67 +
68 +#[cfg(test)]
69 +mod tests {
70 + use std::collections::HashMap;
71 + use std::sync::Arc;
72 +
73 + use llm_multimodal::{MediaContentPart, ModelSpecificValue, PreProcessorConfig};
74 + use ndarray::ArrayD;
75 + use vllm_engine_core_client::protocol::multimodal::{MmField, MmKwargValue};
76 + use vllm_tokenizer::test_utils::TestTokenizer;
77 +
78 + use super::super::{MultimodalModelContext, TokenizerResolver};
79 + use super::*;
80 +
81 + const AUDIO_PAD_ID: u32 = 151_676;
82 +
83 + fn qwen3_asr_info() -> MultimodalModelInfo {
84 + let context = MultimodalModelContext {
85 + model_id: "Qwen/Qwen3-ASR-1.7B".to_string(),
86 + model_type: Some("qwen3_asr".to_string()),
87 + config: serde_json::json!({"model_type": "qwen3_asr"}),
88 + tokenizer: TokenizerResolver(Arc::new(
89 + TestTokenizer::new().with_regular_token("<|audio_pad|>", AUDIO_PAD_ID),
90 + )),
91 + };
92 +
93 + MultimodalModelInfo::from_loaded(
94 + context,
95 + PreProcessorConfig::default(),
96 + PreProcessorConfig::default(),
97 + )
98 + .unwrap()
99 + .expect("Qwen3-ASR multimodal support")
100 + }
101 +
102 + fn wav_i16_mono(sample_rate: u32, samples: &[i16]) -> Vec<u8> {
103 + let data_bytes = samples.len() as u32 * 2;
104 + let mut bytes = Vec::new();
105 + bytes.extend_from_slice(b"RIFF");
106 + bytes.extend_from_slice(&(36 + data_bytes).to_le_bytes());
107 + bytes.extend_from_slice(b"WAVEfmt ");
108 + bytes.extend_from_slice(&16_u32.to_le_bytes());
109 + bytes.extend_from_slice(&1_u16.to_le_bytes());
110 + bytes.extend_from_slice(&1_u16.to_le_bytes());
111 + bytes.extend_from_slice(&sample_rate.to_le_bytes());
112 + bytes.extend_from_slice(&(sample_rate * 2).to_le_bytes());
113 + bytes.extend_from_slice(&2_u16.to_le_bytes());
114 + bytes.extend_from_slice(&16_u16.to_le_bytes());
115 + bytes.extend_from_slice(b"data");
116 + bytes.extend_from_slice(&data_bytes.to_le_bytes());
117 + for sample in samples {
118 + bytes.extend_from_slice(&sample.to_le_bytes());
119 + }
120 + bytes
121 + }
122 +
123 + #[test]
124 + fn resolves_qwen_audio_from_model_spec() {
125 + let info = qwen3_asr_info();
126 + let support = info.audio.as_ref().expect("audio support");
127 +
128 + assert_eq!(
129 + info.placeholder_token(Modality::Audio),
130 + Some("<|audio_pad|>")
131 + );
132 + assert_eq!(support.placeholder.marker_token_id, AUDIO_PAD_ID);
133 + assert_eq!(support.placeholder.embed_token_id, AUDIO_PAD_ID);
134 + assert!(matches!(
135 + support.spec.field_layouts.encoder_input,
136 + llm_multimodal::FieldLayout::Batched
137 + ));
138 +
139 + let preprocessed = PreprocessedEncoderInputs {
140 + encoder_input: ArrayD::zeros(vec![1, 128, 4]),
141 + feature_token_counts: vec![1],
142 + item_sizes: vec![(128, 4)],
143 + model_specific: HashMap::from([
144 + (
145 + "feature_attention_mask".to_string(),
146 + ModelSpecificValue::int_2d(vec![1; 4], 1, 4),
147 + ),
148 + (
149 + "audio_feature_lengths".to_string(),
150 + ModelSpecificValue::int_1d(vec![4]),
151 + ),
152 + ]),
153 + };
154 + let item = item::build_batched_items(
155 + &support.spec,
156 + preprocessed,
157 + vec!["<hash>".to_string()],
158 + vec![None],
159 + ModelDtype::Float32,
160 + )
161 + .unwrap()
162 + .pop()
163 + .unwrap();
164 +
165 + assert!(matches!(
166 + &item.data[AUDIO_PRIMARY_KEY].field,
167 + MmField::Batched(_)
168 + ));
169 + }
170 +
171 + #[tokio::test]
172 + async fn tracker_processor_and_lowering_preserve_audio_contract() {
173 + let info = qwen3_asr_info();
174 + let wav = wav_i16_mono(16_000, &[0; 1_600]);
175 + let expected_hash = llm_multimodal::hasher::hash_audio(&wav);
176 + let fetched = info
177 + .fetch_media(vec![MediaContentPart::AudioData {
178 + data: wav,
179 + mime_type: Some("audio/wav".to_string()),
180 + uuid: Some("audio-1".to_string()),
181 + }])
182 + .await
183 + .unwrap();
184 +
185 + let prepared = info.prepare_audios(fetched.audios, fetched.audio_uuids).await.unwrap();
186 +
187 + assert_eq!(prepared.replacements.len(), 1);
188 + assert!(
189 + prepared.replacements[0]
190 + .tokens
191 + .iter()
192 + .all(|token| *token == AUDIO_PAD_ID as i32)
193 + );
194 + let item = &prepared.items[0];
195 + assert_eq!(item.hash, expected_hash);
196 + assert_eq!(item.uuid.as_deref(), Some("audio-1"));
197 +
198 + let features = &item.data[AUDIO_PRIMARY_KEY];
199 + assert!(matches!(&features.field, MmField::Batched(_)));
200 + assert!(matches!(
201 + features.data.as_ref(),
202 + Some(MmKwargValue::Tensor(tensor))
203 + if tensor.dtype == "float32" && tensor.shape.first() == Some(&128)
204 + ));
205 + let lengths = &item.data["audio_feature_lengths"];
206 + assert!(matches!(&lengths.field, MmField::Batched(_)));
207 + assert!(matches!(
208 + lengths.data.as_ref(),
209 + Some(MmKwargValue::Tensor(tensor))
210 + if tensor.dtype == "int64" && tensor.shape.is_empty()
211 + ));
212 + }
213 +}
rust/src/chat/src/multimodal/expand.rs
+61
@@ -418,6 +418,67 @@ mod tests {
418 assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
419 }
420
421 + const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
422 +
423 + fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
424 + prepared_media(
425 + Modality::Audio,
426 + "<|audio_pad|>",
427 + QWEN3_AUDIO_PAD_ID,
428 + QWEN3_AUDIO_PAD_ID,
429 + replacements,
430 + )
431 + }
432 +
433 + fn qwen3_audio_replacement(tokens: usize) -> PromptReplacement {
434 + PromptReplacement::repeated(
435 + Modality::Audio,
436 + "<|audio_pad|>",
437 + QWEN3_AUDIO_PAD_ID as TokenId,
438 + tokens,
439 + )
440 + }
441 +
442 + #[test]
443 + fn expand_prompt_tokens_interleaves_audio_and_image_prepared_media() {
444 + // Audio marker precedes the image marker: the image range offset must
445 + // account for the earlier audio expansion growth.
446 + let mut prompt_token_ids = vec![1, QWEN3_AUDIO_PAD_ID, 2, LLAMA4_IMAGE_ID, 3];
447 + let prepared = vec![
448 + llama4_prepared(vec![llama4_single_tile_replacement()]),
449 + qwen3_audio_prepared(vec![qwen3_audio_replacement(3)]),
450 + ];
451 +
452 + let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
453 +
454 + assert_eq!(
455 + prompt_token_ids,
456 + vec![
457 + 1,
458 + QWEN3_AUDIO_PAD_ID,
459 + QWEN3_AUDIO_PAD_ID,
460 + QWEN3_AUDIO_PAD_ID,
461 + 2,
462 + LLAMA4_IMAGE_START_ID,
463 + LLAMA4_IMAGE_ID,
464 + LLAMA4_PATCH_ID,
465 + LLAMA4_PATCH_ID,
466 + LLAMA4_IMAGE_END_ID,
467 + 3,
468 + ]
469 + );
470 +
471 + let audio_ranges = &ranges[&Modality::Audio];
472 + assert_eq!(audio_ranges[0].offset, 1);
473 + assert_eq!(audio_ranges[0].length, 3);
474 + assert_bool_mask(&audio_ranges[0], &[true, true, true]);
475 +
476 + let image_ranges = &ranges[&Modality::Image];
477 + assert_eq!(image_ranges[0].offset, 5);
478 + assert_eq!(image_ranges[0].length, 5);
479 + assert_bool_mask(&image_ranges[0], &[false, false, true, true, false]);
480 + }
481 +
482 #[test]
483 fn expand_prompt_tokens_error_names_modality_with_leftover_replacements() {
484 let mut prompt_token_ids = vec![1, QWEN3_IMAGE_PAD_ID, 2];
rust/src/chat/src/multimodal/image.rs
+9 -82
@@ -6,17 +6,15 @@
6
7 use std::sync::Arc;
8
9 -use itertools::izip;
10 -use llm_multimodal::{FieldLayout, ImageFrame, Modality, PreprocessedEncoderInputs};
9 +use llm_multimodal::{ImageFrame, Modality, PreprocessedEncoderInputs};
10 use vllm_engine_core_client::protocol::dtype::ModelDtype;
12 -use vllm_engine_core_client::protocol::multimodal::{
13 - MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice,
14 - SliceSpec,
15 -};
11
17 -use super::{ModalitySupport, MultimodalModelInfo, PreparedItem, PreparedMedia, tensor};
12 +use super::{ModalitySupport, MultimodalModelInfo, PreparedMedia, item};
13 use crate::error::{Error, Result, bail_multimodal, multimodal};
14
15 +/// Forward-kwargs name of the primary image encoder input.
16 +pub(super) const IMAGE_PRIMARY_KEY: &str = "pixel_values";
17 +
18 impl MultimodalModelInfo {
19 /// Preprocess all fetched image frames as one batch and build per-item
20 /// features.
@@ -30,9 +28,7 @@ impl MultimodalModelInfo {
28 modality: Modality::Image.to_string(),
29 })?;
30 let preprocessed = self.preprocess_images(support, &frames).await?;
33 - let replacements =
34 - self.spec
35 - .prompt_replacements_for(&self.context, &preprocessed, Modality::Image)?;
31 + let replacements = support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
32 if replacements.len() != frames.len() {
33 bail_multimodal!(
34 "number of image prompt replacements {} does not match number of images {}",
@@ -40,7 +36,9 @@ impl MultimodalModelInfo {
36 frames.len()
37 );
38 }
43 - let items = self.build_image_items(preprocessed, &frames, uuids, model_dtype)?;
39 + let hashes = frames.iter().map(|frame| frame.hash.clone()).collect();
40 + let items =
41 + item::build_batched_items(&support.spec, preprocessed, hashes, uuids, model_dtype)?;
42
43 Ok(PreparedMedia {
44 modality: Modality::Image,
@@ -70,75 +68,4 @@ impl MultimodalModelInfo {
68 .await
69 .map_err(|error| multimodal!("image preprocessing task failed: {error}"))?
70 }
73 -
74 - /// Convert one batch of preprocessed image tensors into per-item engine
75 - /// kwargs.
76 - ///
77 - /// Tensor fields are sliced per item according to the model spec's field
78 - /// layout declarations.
79 - fn build_image_items(
80 - &self,
81 - preprocessed: PreprocessedEncoderInputs,
82 - frames: &[Arc<ImageFrame>],
83 - uuids: Vec<Option<String>>,
84 - model_dtype: ModelDtype,
85 - ) -> Result<Vec<PreparedItem>> {
86 - let len = frames.len();
87 - let tensors = tensor::collect_tensors(preprocessed, "pixel_values", model_dtype)?;
88 -
89 - let mut items = Vec::with_capacity(len);
90 - for (index, (frame, uuid)) in izip!(frames, uuids).enumerate() {
91 - let mut data = MmKwargsItem::new();
92 - for (key, tensor) in &tensors {
93 - let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(key);
94 - let (value, field) = match self.spec.field_layouts.get(key) {
95 - Some(FieldLayout::Batched) => (
96 - tensor.batched_value_at(index)?,
97 - MmField::Batched(MmBatchedField { keep_on_cpu }),
98 - ),
99 - Some(FieldLayout::Flat { sizes_key }) => {
100 - let sizes = tensors.get(sizes_key).ok_or_else(|| {
101 - multimodal!("flat tensor sizes key `{sizes_key}` is missing")
102 - })?;
103 - let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?;
104 - (
105 - tensor.flat_value_range(start, end)?,
106 - MmField::Flat(MmFlatField {
107 - slices: vec![MmSlice::Slice(SliceSpec {
108 - start: Some(0),
109 - stop: Some((end - start) as isize),
110 - step: None,
111 - })],
112 - dim: 0,
113 - keep_on_cpu,
114 - }),
115 - )
116 - }
117 - None => (
118 - tensor.clone(),
119 - MmField::Shared(MmSharedField {
120 - batch_size: len,
121 - keep_on_cpu,
122 - }),
123 - ),
124 - };
125 -
126 - data.insert(
127 - key.clone(),
128 - MmFieldElem {
129 - data: Some(value.try_into()?),
130 - field,
131 - },
132 - );
133 - }
134 -
135 - items.push(PreparedItem {
136 - data,
137 - hash: frame.hash.clone(),
138 - uuid,
139 - });
140 - }
141 -
142 - Ok(items)
143 - }
71 }
rust/src/chat/src/multimodal/item.rs new
+84
@@ -0,0 +1,84 @@
1 +// SPDX-License-Identifier: Apache-2.0
2 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +//! Shared lowering from batched preprocessor output to per-item engine kwargs.
5 +
6 +use itertools::izip;
7 +use llm_multimodal::{FieldLayout, PreprocessedEncoderInputs};
8 +use vllm_engine_core_client::protocol::dtype::ModelDtype;
9 +use vllm_engine_core_client::protocol::multimodal::{
10 + MmBatchedField, MmField, MmFieldElem, MmFlatField, MmKwargsItem, MmSharedField, MmSlice,
11 + SliceSpec,
12 +};
13 +
14 +use super::{PreparedItem, ResolvedMultimodalSpec, tensor};
15 +use crate::error::{Result, bail_multimodal, multimodal};
16 +
17 +/// Split one batch of preprocessed tensors into engine kwargs per media item.
18 +pub(super) fn build_batched_items(
19 + spec: &ResolvedMultimodalSpec,
20 + preprocessed: PreprocessedEncoderInputs,
21 + hashes: Vec<String>,
22 + uuids: Vec<Option<String>>,
23 + float_dtype: ModelDtype,
24 +) -> Result<Vec<PreparedItem>> {
25 + let len = hashes.len();
26 + if uuids.len() != len {
27 + bail_multimodal!(
28 + "number of media UUIDs {} does not match number of media items {len}",
29 + uuids.len()
30 + );
31 + }
32 + let tensors = tensor::collect_tensors(preprocessed, spec.primary_key(), float_dtype)?;
33 +
34 + let mut items = Vec::with_capacity(len);
35 + for (index, (hash, uuid)) in izip!(hashes, uuids).enumerate() {
36 + let mut data = MmKwargsItem::new();
37 + for (key, tensor) in &tensors {
38 + let keep_on_cpu = spec.keep_on_cpu_keys.contains(key);
39 + let (value, field) = match spec.field_layout_for(key) {
40 + Some(FieldLayout::Batched) => (
41 + tensor.batched_value_at(index)?,
42 + MmField::Batched(MmBatchedField { keep_on_cpu }),
43 + ),
44 + Some(FieldLayout::Flat { sizes_key }) => {
45 + let sizes = tensors.get(sizes_key).ok_or_else(|| {
46 + multimodal!("flat tensor sizes key `{sizes_key}` is missing")
47 + })?;
48 + let (start, end) = tensor::flat_range_for_index(sizes, sizes_key, index)?;
49 + (
50 + tensor.flat_value_range(start, end)?,
51 + MmField::Flat(MmFlatField {
52 + slices: vec![MmSlice::Slice(SliceSpec {
53 + start: Some(0),
54 + stop: Some((end - start) as isize),
55 + step: None,
56 + })],
57 + dim: 0,
58 + keep_on_cpu,
59 + }),
60 + )
61 + }
62 + None => (
63 + tensor.clone(),
64 + MmField::Shared(MmSharedField {
65 + batch_size: len,
66 + keep_on_cpu,
67 + }),
68 + ),
69 + };
70 +
71 + data.insert(
72 + key.clone(),
73 + MmFieldElem {
74 + data: Some(value.try_into()?),
75 + field,
76 + },
77 + );
78 + }
79 +
80 + items.push(PreparedItem { data, hash, uuid });
81 + }
82 +
83 + Ok(items)
84 +}
rust/src/chat/src/multimodal/video.rs
+56 -59
@@ -23,7 +23,7 @@ use crate::error::{Error, Result, bail_multimodal, multimodal};
23 ///
24 /// Video-capable vLLM models read `pixel_values_videos` alongside
25 /// `video_grid_thw`, mirroring the HF processor output naming.
26 -const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos";
26 +pub(super) const VIDEO_PRIMARY_KEY: &str = "pixel_values_videos";
27
28 impl MultimodalModelInfo {
29 /// Preprocess fetched video clips one at a time and build per-item
@@ -47,8 +47,7 @@ impl MultimodalModelInfo {
47 for (clip, uuid) in izip!(&clips, uuids) {
48 let preprocessed = self.preprocess_video_clip(support, Arc::clone(clip)).await?;
49 let mut clip_replacements =
50 - self.spec
51 - .prompt_replacements_for(&self.context, &preprocessed, Modality::Video)?;
50 + support.spec.prompt_replacements_for(&self.context, &preprocessed)?;
51 if clip_replacements.len() != 1 {
52 bail_multimodal!(
53 "expected exactly one prompt replacement per video clip, got {}",
@@ -56,7 +55,8 @@ impl MultimodalModelInfo {
55 );
56 }
57 replacements.push(clip_replacements.pop().unwrap());
59 - items.push(self.build_video_item(
58 + items.push(build_video_item(
59 + support,
60 preprocessed,
61 clip.hash.clone(),
62 uuid,
@@ -107,30 +107,36 @@ impl MultimodalModelInfo {
107 .await
108 .map_err(|error| multimodal!("video preprocessing task failed: {error}"))?
109 }
110 +}
111
111 - /// Convert one preprocessed video clip into engine kwargs.
112 - ///
113 - /// The clip is a batch of one, so no per-item slicing is required: the
114 - /// primary tensor ships as a full-range flat field (the engine re-batches
115 - /// flat fields by concatenating along the declared dim, matching vLLM's
116 - /// `flat_from_sizes` treatment of video patches), and batched metadata
117 - /// tensors drop their singleton batch axis.
118 - fn build_video_item(
119 - &self,
120 - preprocessed: PreprocessedEncoderInputs,
121 - hash: String,
122 - uuid: Option<String>,
123 - model_dtype: ModelDtype,
124 - ) -> Result<PreparedItem> {
125 - let tensors = tensor::collect_tensors(preprocessed, VIDEO_PRIMARY_KEY, model_dtype)?;
112 +/// Convert one preprocessed video clip into engine kwargs.
113 +///
114 +/// The clip is a batch of one, so no per-item slicing is required: the
115 +/// primary tensor ships as a full-range flat field (the engine re-batches
116 +/// flat fields by concatenating along the declared dim, matching vLLM's
117 +/// `flat_from_sizes` treatment of video patches), and batched metadata
118 +/// tensors drop their singleton batch axis.
119 +fn build_video_item(
120 + support: &ModalitySupport,
121 + preprocessed: PreprocessedEncoderInputs,
122 + hash: String,
123 + uuid: Option<String>,
124 + model_dtype: ModelDtype,
125 +) -> Result<PreparedItem> {
126 + let tensors = tensor::collect_tensors(preprocessed, support.spec.primary_key(), model_dtype)?;
127
127 - let mut data = MmKwargsItem::new();
128 - for (key, tensor) in tensors {
129 - let keep_on_cpu = self.spec.keep_on_cpu_keys.contains(&key);
130 - let (value, field) = if key == VIDEO_PRIMARY_KEY {
128 + let mut data = MmKwargsItem::new();
129 + for (key, tensor) in tensors {
130 + let keep_on_cpu = support.spec.keep_on_cpu_keys.contains(&key);
131 + let (value, field) = match support.spec.field_layout_for(&key) {
132 + Some(FieldLayout::Batched) => (
133 + tensor.batched_value_at(0)?,
134 + MmField::Batched(MmBatchedField { keep_on_cpu }),
135 + ),
136 + Some(FieldLayout::Flat { .. }) => {
137 let len = tensor
138 .first_dim()
133 - .ok_or_else(|| multimodal!("video encoder input `{key}` is not a tensor"))?;
139 + .ok_or_else(|| multimodal!("flat video input `{key}` is not a tensor"))?;
140 (
141 tensor,
142 MmField::Flat(MmFlatField {
@@ -143,35 +149,26 @@ impl MultimodalModelInfo {
149 keep_on_cpu,
150 }),
151 )
146 - } else if matches!(
147 - self.spec.field_layouts.get(&key),
148 - Some(FieldLayout::Batched)
149 - ) {
150 - (
151 - tensor.batched_value_at(0)?,
152 - MmField::Batched(MmBatchedField { keep_on_cpu }),
153 - )
154 - } else {
155 - (
156 - tensor,
157 - MmField::Shared(MmSharedField {
158 - batch_size: 1,
159 - keep_on_cpu,
160 - }),
161 - )
162 - };
163 -
164 - data.insert(
165 - key,
166 - MmFieldElem {
167 - data: Some(value.try_into()?),
168 - field,
169 - },
170 - );
171 - }
152 + }
153 + None => (
154 + tensor,
155 + MmField::Shared(MmSharedField {
156 + batch_size: 1,
157 + keep_on_cpu,
158 + }),
159 + ),
160 + };
161
173 - Ok(PreparedItem { data, hash, uuid })
162 + data.insert(
163 + key,
164 + MmFieldElem {
165 + data: Some(value.try_into()?),
166 + field,
167 + },
168 + );
169 }
170 +
171 + Ok(PreparedItem { data, hash, uuid })
172 }
173
174 #[cfg(test)]
@@ -287,14 +284,14 @@ mod tests {
284 ]),
285 };
286
290 - let item = info
291 - .build_video_item(
292 - preprocessed,
293 - "<hash>".to_string(),
294 - None,
295 - ModelDtype::Float32,
296 - )
297 - .unwrap();
287 + let item = build_video_item(
288 + info.video.as_ref().unwrap(),
289 + preprocessed,
290 + "<hash>".to_string(),
291 + None,
292 + ModelDtype::Float32,
293 + )
294 + .unwrap();
295
296 let primary = &item.data[VIDEO_PRIMARY_KEY];
297 assert!(matches!(
rust/src/chat/src/renderer/hf/mod.rs
+58 -1
@@ -42,6 +42,7 @@ pub use self::format::ChatTemplateContentFormatOption;
42 pub struct MultimodalRenderInfo {
43 pub image_token: Option<String>,
44 pub video_token: Option<String>,
45 + pub audio_token: Option<String>,
46 }
47
48 /// Hugging Face chat-template renderer backed by the local Jinja chat-template
@@ -263,6 +264,7 @@ enum TemplateContentPart {
264 Text { text: String },
265 Image,
266 Video,
267 + Audio,
268 }
269
270 #[derive(Debug, Serialize)]
@@ -437,6 +439,12 @@ fn to_template_openai_content(
439 .ok_or(Error::UnsupportedMultimodalContent("video_url"))?;
440 Ok(TemplateContentPart::Video)
441 }
442 + ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => {
443 + multimodal
444 + .and_then(|multimodal| multimodal.audio_token.as_ref())
445 + .ok_or(Error::UnsupportedMultimodalContent("audio"))?;
446 + Ok(TemplateContentPart::Audio)
447 + }
448 })
449 .collect(),
450 }
@@ -465,6 +473,12 @@ fn to_template_string_content(
473 .ok_or(Error::UnsupportedMultimodalContent("video_url"))?;
474 out.push_str(video_token);
475 }
476 + ChatContentPart::InputAudio { .. } | ChatContentPart::AudioUrl { .. } => {
477 + let audio_token = multimodal
478 + .and_then(|multimodal| multimodal.audio_token.as_ref())
479 + .ok_or(Error::UnsupportedMultimodalContent("audio"))?;
480 + out.push_str(audio_token);
481 + }
482 }
483 }
484 Ok(out)
@@ -492,7 +506,9 @@ fn append_continue_final_message_tag(message: &mut TemplateMessage) -> Result<St
506 // Pick the last text part in the message.
507 TemplateContent::OpenAi(parts) => parts.iter_mut().rev().find_map(|part| match part {
508 TemplateContentPart::Text { text } => Some(text),
495 - TemplateContentPart::Image | TemplateContentPart::Video => None,
509 + TemplateContentPart::Image
510 + | TemplateContentPart::Video
511 + | TemplateContentPart::Audio => None,
512 }),
513 };
514 let text = text.ok_or_else(|| {
@@ -603,6 +619,7 @@ mod tests {
619 .with_multimodal(Some(MultimodalRenderInfo {
620 image_token: Some("<image>".to_string()),
621 video_token: Some("<video>".to_string()),
622 + audio_token: Some("<audio>".to_string()),
623 }))
624 .render(request)
625 }
@@ -623,6 +640,15 @@ mod tests {
640 ])])
641 }
642
643 + fn audio_request() -> ChatRequest {
644 + sample_request(vec![ChatMessage::user(vec![
645 + ChatContentPart::text("a"),
646 + ChatContentPart::input_audio("dGVzdA==", Some("wav".to_string())),
647 + ChatContentPart::audio_url("https://example.com/demo.mp3"),
648 + ChatContentPart::text("b"),
649 + ])])
650 + }
651 +
652 #[test]
653 fn string_content_format_replaces_image_with_placeholder_text() {
654 let rendered = render_mm(
@@ -647,6 +673,21 @@ mod tests {
673 assert_eq!(rendered.prompt, Prompt::Text("a<video>b".to_string()));
674 }
675
676 + #[test]
677 + fn string_content_format_replaces_audio_with_placeholder_text() {
678 + let rendered = render_mm(
679 + "{{ messages[0].content }}",
680 + &audio_request(),
681 + ChatTemplateContentFormatOption::String,
682 + )
683 + .unwrap();
684 +
685 + assert_eq!(
686 + rendered.prompt,
687 + Prompt::Text("a<audio><audio>b".to_string())
688 + );
689 + }
690 +
691 #[test]
692 fn openai_content_format_normalizes_image_url_for_template() {
693 let rendered = render_mm(
@@ -671,6 +712,21 @@ mod tests {
712 assert_eq!(rendered.prompt, Prompt::Text("a<|video_pad|>b".to_string()));
713 }
714
715 + #[test]
716 + fn openai_content_format_normalizes_audio_for_template() {
717 + let rendered = render_mm(
718 + "{% for item in messages[0].content %}{% if item.type == 'audio' %}<|audio_pad|>{% else %}{{ item.text }}{% endif %}{% endfor %}",
719 + &audio_request(),
720 + ChatTemplateContentFormatOption::OpenAi,
721 + )
722 + .unwrap();
723 +
724 + assert_eq!(
725 + rendered.prompt,
726 + Prompt::Text("a<|audio_pad|><|audio_pad|>b".to_string())
727 + );
728 + }
729 +
730 #[test]
731 fn video_parts_are_rejected_when_model_lacks_video_support() {
732 let error = HfChatRenderer::new(
@@ -682,6 +738,7 @@ mod tests {
738 .with_multimodal(Some(MultimodalRenderInfo {
739 image_token: Some("<image>".to_string()),
740 video_token: None,
741 + audio_token: None,
742 }))
743 .render(&video_request())
744 .unwrap_err();
rust/src/chat/src/request.rs
+34 -1
@@ -44,6 +44,17 @@ pub enum ChatContentPart {
44 video_url: String,
45 uuid: Option<String>,
46 },
47 + /// One `input_audio` content block carrying base64-encoded audio bytes.
48 + InputAudio {
49 + data: String,
50 + format: Option<String>,
51 + uuid: Option<String>,
52 + },
53 + /// One audio URL/data URL content block.
54 + AudioUrl {
55 + audio_url: String,
56 + uuid: Option<String>,
57 + },
58 // ImageData...
59 // VideoData...
60 // ImageEmbeds...
@@ -72,6 +83,23 @@ impl ChatContentPart {
83 }
84 }
85
86 + /// Construct one base64-encoded input-audio content part.
87 + pub fn input_audio(data: impl Into<String>, format: Option<String>) -> Self {
88 + Self::InputAudio {
89 + data: data.into(),
90 + format,
91 + uuid: None,
92 + }
93 + }
94 +
95 + /// Construct one audio URL content part with the given URL string.
96 + pub fn audio_url(audio_url: impl Into<String>) -> Self {
97 + Self::AudioUrl {
98 + audio_url: audio_url.into(),
99 + uuid: None,
100 + }
101 + }
102 +
103 /// Return the text content of this part when it's a text block, or an
104 /// "unsupported multimodal content" error otherwise.
105 pub(crate) fn as_text(&self) -> Result<&str> {
@@ -79,6 +107,8 @@ impl ChatContentPart {
107 Self::Text { text } => Ok(text),
108 Self::ImageUrl { .. } => Err(Error::UnsupportedMultimodalContent("image_url")),
109 Self::VideoUrl { .. } => Err(Error::UnsupportedMultimodalContent("video_url")),
110 + Self::InputAudio { .. } => Err(Error::UnsupportedMultimodalContent("input_audio")),
111 + Self::AudioUrl { .. } => Err(Error::UnsupportedMultimodalContent("audio_url")),
112 }
113 }
114
@@ -91,7 +121,10 @@ impl ChatContentPart {
121 pub(crate) fn is_multimodal(&self) -> bool {
122 match self {
123 Self::Text { .. } => false,
94 - Self::ImageUrl { .. } | Self::VideoUrl { .. } => true,
124 + Self::ImageUrl { .. }
125 + | Self::VideoUrl { .. }
126 + | Self::InputAudio { .. }
127 + | Self::AudioUrl { .. } => true,
128 }
129 }
130 }
rust/src/server/src/routes/openai/chat_completions/convert.rs
+56 -2
@@ -299,6 +299,15 @@ fn convert_content(content: MessageContent) -> Result<ChatContent, ApiError> {
299 video_url: video_url.url,
300 uuid,
301 }),
302 + ContentPart::AudioUrl { audio_url, uuid } => Ok(ChatContentPart::AudioUrl {
303 + audio_url: audio_url.url,
304 + uuid,
305 + }),
306 + ContentPart::InputAudio { input_audio, uuid } => Ok(ChatContentPart::InputAudio {
307 + data: input_audio.data,
308 + format: input_audio.format,
309 + uuid,
310 + }),
311 })
312 .try_collect()
313 .map(ChatContent::Parts),
@@ -402,8 +411,8 @@ mod tests {
411 AssistantRole, ChatCompletionMessage, ChatCompletionRequest,
412 };
413 use crate::routes::openai::utils::types::{
405 - ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, MessageContent,
406 - StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
414 + AudioUrl, ChatMessage, ContentPart, Function, FunctionCallResponse, ImageUrl, InputAudio,
415 + MessageContent, StreamOptions, Tool, ToolCall, ToolChoice, ToolChoiceValue, VideoUrl,
416 };
417 use crate::utils::{ResolvedRequestContext, resolve_request_context};
418
@@ -810,6 +819,51 @@ mod tests {
819 );
820 }
821
822 + #[test]
823 + fn prepare_chat_request_accepts_audio_content_parts() {
824 + let request = ChatCompletionRequest {
825 + model: "Qwen/Qwen3-ASR-1.7B".to_string(),
826 + messages: vec![ChatMessage::User {
827 + content: MessageContent::Parts(vec![
828 + ContentPart::InputAudio {
829 + input_audio: InputAudio {
830 + data: "dGVzdA==".to_string(),
831 + format: Some("wav".to_string()),
832 + },
833 + uuid: Some("audio-1".to_string()),
834 + },
835 + ContentPart::AudioUrl {
836 + audio_url: AudioUrl {
837 + url: "https://example.com/audio.mp3".to_string(),
838 + },
839 + uuid: None,
840 + },
841 + ]),
842 + name: None,
843 + }],
844 + ..base_request()
845 + };
846 +
847 + let prepared = prepare_chat_request(
848 + request,
849 + &served(&["Qwen/Qwen3-ASR-1.7B"]),
850 + ResolvedRequestContext::default(),
851 + )
852 + .expect("request is valid");
853 +
854 + assert_eq!(
855 + prepared.chat_request.messages,
856 + vec![VllmChatMessage::user(vec![
857 + ChatContentPart::InputAudio {
858 + data: "dGVzdA==".to_string(),
859 + format: Some("wav".to_string()),
860 + uuid: Some("audio-1".to_string()),
861 + },
862 + ChatContentPart::audio_url("https://example.com/audio.mp3"),
863 + ])]
864 + );
865 + }
866 +
867 #[test]
868 fn prepare_chat_request_rejects_assistant_image_url_content_parts() {
869 let request = ChatCompletionRequest {
rust/src/server/src/routes/openai/utils/types.rs
+25
@@ -99,6 +99,18 @@ pub enum ContentPart {
99 #[serde(skip_serializing_if = "Option::is_none")]
100 uuid: Option<String>,
101 },
102 + #[serde(rename = "audio_url")]
103 + AudioUrl {
104 + audio_url: AudioUrl,
105 + #[serde(skip_serializing_if = "Option::is_none")]
106 + uuid: Option<String>,
107 + },
108 + #[serde(rename = "input_audio")]
109 + InputAudio {
110 + input_audio: InputAudio,
111 + #[serde(skip_serializing_if = "Option::is_none")]
112 + uuid: Option<String>,
113 + },
114 }
115
116 #[serde_with::skip_serializing_none]
@@ -113,6 +125,19 @@ pub struct VideoUrl {
125 pub url: String,
126 }
127
128 +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
129 +pub struct AudioUrl {
130 + pub url: String,
131 +}
132 +
133 +/// Base64-encoded audio bytes in OpenAI `input_audio` form.
134 +#[serde_with::skip_serializing_none]
135 +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
136 +pub struct InputAudio {
137 + pub data: String,
138 + pub format: Option<String>,
139 +}
140 +
141 // ============================================================================
142 // Streaming
143 // ============================================================================
rust/src/server/src/routes/tests.rs
+4 -3
@@ -551,9 +551,10 @@ fn render_fake_content(content: &ChatContent, placeholder: &str) -> vllm_chat::R
551 for part in parts {
552 match part {
553 ChatContentPart::Text { text } => out.push_str(text),
554 - ChatContentPart::ImageUrl { .. } | ChatContentPart::VideoUrl { .. } => {
555 - out.push_str(placeholder)
556 - }
554 + ChatContentPart::ImageUrl { .. }
555 + | ChatContentPart::VideoUrl { .. }
556 + | ChatContentPart::InputAudio { .. }
557 + | ChatContentPart::AudioUrl { .. } => out.push_str(placeholder),
558 }
559 }
560 out
vllm/model_executor/models/qwen2_5_omni_thinker.py
+28
@@ -114,6 +114,23 @@ except (ImportError, ModuleNotFoundError):
114 logger = init_logger(__name__)
115
116
117 +def unpad_and_flat_audio_features(
118 + input_audio_features: torch.Tensor,
119 + audio_feature_lengths: torch.Tensor,
120 +) -> torch.Tensor:
121 + """Unpad and flatten batched audio features."""
122 + if len(audio_feature_lengths) != input_audio_features.size(0):
123 + raise ValueError(
124 + "Length of audio_feature_lengths must match "
125 + "the batch size of input_audio_features."
126 + )
127 + unpadded_features = [
128 + audio_feature[..., :length]
129 + for audio_feature, length in zip(input_audio_features, audio_feature_lengths)
130 + ]
131 + return torch.cat(unpadded_features, dim=-1)
132 +
133 +
134 def check_interleaved_audio_video(
135 is_video: torch.Tensor,
136 is_audio: torch.Tensor,
@@ -887,6 +904,17 @@ class Qwen2_5OmniConditionalGenerationMixin:
904 if input_audio_features is None:
905 return None
906
907 + # inputs features from rust frontend is batched and padded
908 + # with shape [batch_size, n_mels, padded_seq_len], different
909 + # from python's shape [n_mels, batch_size * seq_len]
910 + if (
911 + isinstance(input_audio_features, torch.Tensor)
912 + and input_audio_features.dim() == 3
913 + ):
914 + input_audio_features = unpad_and_flat_audio_features(
915 + input_audio_features, audio_feature_lengths
916 + )
917 +
918 return Qwen2_5OmniAudioFeatureInputs(
919 type="audio_features",
920 input_features=input_audio_features,
vllm/model_executor/models/qwen3_asr.py
+20 -1
@@ -46,9 +46,12 @@ from vllm.model_executor.models.interfaces import (
46 SupportsTranscription,
47 )
48 from vllm.model_executor.models.module_mapping import MultiModelKeys
49 +from vllm.model_executor.models.qwen2_5_omni_thinker import (
50 + Qwen2_5OmniAudioFeatureInputs,
51 + unpad_and_flat_audio_features,
52 +)
53 from vllm.model_executor.models.qwen3 import Qwen3ForCausalLM
54 from vllm.model_executor.models.qwen3_omni_moe_thinker import (
51 - Qwen2_5OmniAudioFeatureInputs,
55 Qwen3OmniMoeAudioEncoder,
56 Qwen3OmniMoeThinkerMultiModalProcessor,
57 )
@@ -363,6 +366,11 @@ class Qwen3ASRForConditionalGeneration(
366 "thinker.lm_head.": "language_model.lm_head.",
367 "thinker.model.": "language_model.model.",
368 "thinker.": "",
369 + # HF format mapper
370 + "model.audio_tower.": "audio_tower.",
371 + "model.language_model.": "language_model.model.",
372 + "model.multi_modal_projector.linear_1.": "audio_tower.proj1.",
373 + "model.multi_modal_projector.linear_2.": "audio_tower.proj2.",
374 }
375 )
376
@@ -412,6 +420,17 @@ class Qwen3ASRForConditionalGeneration(
420 if input_audio_features is None:
421 return None
422
423 + # inputs features from rust frontend is batched and padded
424 + # with shape [batch_size, n_mels, padded_seq_len], different
425 + # from python's shape [n_mels, batch_size * seq_len]
426 + if (
427 + isinstance(input_audio_features, torch.Tensor)
428 + and input_audio_features.dim() == 3
429 + ):
430 + input_audio_features = unpad_and_flat_audio_features(
431 + input_audio_features, audio_feature_lengths
432 + )
433 +
434 return Qwen2_5OmniAudioFeatureInputs(
435 type="audio_features",
436 input_features=input_audio_features,
vllm/transformers_utils/configs/qwen3_asr.py
+17 -4
@@ -407,16 +407,29 @@ class Qwen3ASRConfig(PretrainedConfig):
407 def __init__(
408 self,
409 thinker_config=None,
410 + audio_config=None,
411 + text_config=None,
412 + audio_token_id=None,
413 support_languages=None,
414 timestamp_token_id=None,
415 timestamp_segment_time=None,
416 **kwargs,
417 ):
418 if thinker_config is None:
416 - thinker_config = {}
417 - logger.info(
418 - "thinker_config is None. Initializing thinker model with default values"
419 - )
419 + thinker_config = {
420 + key: value
421 + for key, value in (
422 + ("audio_config", audio_config),
423 + ("text_config", text_config),
424 + ("audio_token_id", audio_token_id),
425 + )
426 + if value is not None
427 + }
428 + if not thinker_config:
429 + logger.info(
430 + "thinker_config is None. Initializing thinker model with "
431 + "default values"
432 + )
433
434 self.thinker_config = Qwen3ASRThinkerConfig(**thinker_config)
435 self.support_languages = support_languages