[Rust Frontend][gRPC] Add audio and video media inputs (#53760)
Signed-off-by: Connor Carpenter <connorc@nvidia.com>
Connor Carpenter committed
Aug 29, 2026 at 23:31 UTC
7a67941c1d0506d0690fbb4e91b573c660d24981
4 files changed
+122
-48
rust/src/chat/src/multimodal/expand.rs
+41
-8
@@ -86,20 +86,30 @@ pub(super) fn expand_prompt_token_ids(
86
}
87
88
let replacement_len = replacement.tokens.len();
89
+ let structural_prefix = replacement.structural_prefix;
90
+ if structural_prefix > expanded.len() {
91
+ bail_multimodal!(
92
+ "placeholder token `{}` declares a structural prefix of {} token(s), but only {} token(s) precede it",
93
+ lane.placeholder_token,
94
+ structural_prefix,
95
+ expanded.len()
96
+ );
97
+ }
98
+ let range_len = replacement_len.saturating_add(structural_prefix);
99
let is_embed = {
90
- let mask = replacement
91
- .tokens
92
- .iter()
93
- .map(|&token| token as u32 == lane.embed_token_id)
94
- .collect::<Vec<_>>();
95
- WireTensor::from_bool(vec![replacement_len], mask).map_err(Error::Multimodal)?
100
+ let mut mask = Vec::with_capacity(range_len);
101
+ mask.extend(std::iter::repeat_n(false, structural_prefix));
102
+ mask.extend(
103
+ replacement.tokens.iter().map(|&token| token as u32 == lane.embed_token_id),
104
+ );
105
+ WireTensor::from_bool(vec![range_len], mask).map_err(Error::Multimodal)?
106
};
107
98
- let expanded_offset = expanded.len();
108
+ let expanded_offset = expanded.len() - structural_prefix;
109
expanded.extend(replacement.tokens.iter().map(|&token| token as u32));
110
ranges.entry(lane.modality).or_default().push(PlaceholderRange {
111
offset: expanded_offset,
102
- length: replacement_len,
112
+ length: range_len,
113
is_embed: Some(is_embed),
114
});
115
}
@@ -420,6 +430,29 @@ mod tests {
430
assert_bool_mask(&video_ranges[0], &[true, true, true, true]);
431
}
432
433
+ #[test]
434
+ fn expand_prompt_tokens_includes_structural_prefix_in_video_range() {
435
+ const VISION_START_ID: u32 = 151652;
436
+ const VISION_END_ID: u32 = 151653;
437
+
438
+ let mut prompt_token_ids = vec![1, VISION_START_ID, QWEN3_VIDEO_PAD_ID, VISION_END_ID, 2];
439
+ let replacement = PromptReplacement::repeated(
440
+ Modality::Video,
441
+ "<|video_pad|>",
442
+ QWEN3_VIDEO_PAD_ID as TokenId,
443
+ 3,
444
+ )
445
+ .with_structural_prefix(1);
446
+ let prepared = vec![qwen3_video_prepared(vec![replacement])];
447
+
448
+ let ranges = expand_prompt_token_ids(&mut prompt_token_ids, &prepared).unwrap();
449
+
450
+ let range = &ranges[&Modality::Video][0];
451
+ assert_eq!(range.offset, 1);
452
+ assert_eq!(range.length, 4);
453
+ assert_bool_mask(range, &[false, true, true, true]);
454
+ }
455
+
456
const QWEN3_AUDIO_PAD_ID: u32 = 151_676;
457
458
fn qwen3_audio_prepared(replacements: Vec<PromptReplacement>) -> PreparedMedia {
rust/src/server/src/grpc/convert.rs
+50
-33
@@ -22,49 +22,66 @@ pub fn media_parts_from_request(
22
) -> Result<Vec<MediaContentPart>, Status> {
23
let mut parts = Vec::with_capacity(media.len());
24
for (index, item) in media.into_iter().enumerate() {
25
- match item.modality() {
26
- pb::Modality::Image => {}
27
- pb::Modality::Unspecified => {
28
- return Err(Status::invalid_argument(format!(
29
- "media[{index}].modality is required"
30
- )));
31
- }
32
- other => {
33
- return Err(Status::unimplemented(format!(
34
- "media[{index}].modality {other:?} is not supported by the gRPC service"
35
- )));
36
- }
25
+ let modality = item.modality();
26
+ if modality == pb::Modality::Unspecified {
27
+ return Err(Status::invalid_argument(format!(
28
+ "media[{index}].modality is required"
29
+ )));
30
}
31
let uuid = (!item.uuid.is_empty()).then_some(item.uuid);
32
let mime_type = (!item.mime_type.is_empty()).then_some(item.mime_type);
40
- let part = match item.source {
41
- Some(pb::media_item::Source::Url(url)) => {
42
- validate_media_uri(index, "url", &url, &["http", "https"])?;
43
- MediaContentPart::ImageUrl {
44
- url,
45
- detail: None,
33
+ let source = item.source.ok_or_else(|| {
34
+ Status::invalid_argument(format!("media[{index}].source is required"))
35
+ })?;
36
+ match &source {
37
+ pb::media_item::Source::Url(url) => {
38
+ validate_media_uri(index, "url", url, &["http", "https"])?;
39
+ }
40
+ pb::media_item::Source::DataUri(uri) => {
41
+ validate_media_uri(index, "data_uri", uri, &["data"])?;
42
+ }
43
+ pb::media_item::Source::RawBytes(_) => {}
44
+ }
45
+ let part = match (modality, source) {
46
+ (
47
+ pb::Modality::Image,
48
+ pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
49
+ ) => MediaContentPart::ImageUrl {
50
+ url,
51
+ detail: None,
52
+ uuid,
53
+ },
54
+ (pb::Modality::Image, pb::media_item::Source::RawBytes(data)) => {
55
+ MediaContentPart::ImageData {
56
+ data,
57
+ mime_type,
58
uuid,
59
+ detail: None,
60
}
61
}
49
- Some(pb::media_item::Source::DataUri(uri)) => {
50
- validate_media_uri(index, "data_uri", &uri, &["data"])?;
51
- MediaContentPart::ImageUrl {
52
- url: uri,
53
- detail: None,
62
+ (
63
+ pb::Modality::Video,
64
+ pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
65
+ ) => MediaContentPart::VideoUrl { url, uuid },
66
+ (pb::Modality::Video, pb::media_item::Source::RawBytes(data)) => {
67
+ MediaContentPart::VideoData {
68
+ data,
69
+ mime_type,
70
uuid,
71
}
72
}
57
- Some(pb::media_item::Source::RawBytes(bytes)) => MediaContentPart::ImageData {
58
- data: bytes,
59
- mime_type,
60
- uuid,
61
- detail: None,
62
- },
63
- None => {
64
- return Err(Status::invalid_argument(format!(
65
- "media[{index}].source is required"
66
- )));
73
+ (
74
+ pb::Modality::Audio,
75
+ pb::media_item::Source::Url(url) | pb::media_item::Source::DataUri(url),
76
+ ) => MediaContentPart::AudioUrl { url, uuid },
77
+ (pb::Modality::Audio, pb::media_item::Source::RawBytes(data)) => {
78
+ MediaContentPart::AudioData {
79
+ data,
80
+ mime_type,
81
+ uuid,
82
+ }
83
}
84
+ (pb::Modality::Unspecified, _) => unreachable!("modality validated above"),
85
};
86
parts.push(part);
87
}
tests/model_executor/test_qwen3_asr_mrope.py
new
+27
@@ -0,0 +1,27 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import torch
5
+
6
+from vllm.model_executor.models.qwen3_asr import (
7
+ Qwen3ASRForConditionalGeneration,
8
+)
9
+from vllm.multimodal.inputs import MultiModalFeatureSpec, PlaceholderRange
10
+
11
+
12
+def test_mrope_positions_with_stripped_audio_data() -> None:
13
+ input_tokens = list(range(7))
14
+ feature = MultiModalFeatureSpec(
15
+ data=None,
16
+ modality="audio",
17
+ identifier="audio",
18
+ mm_position=PlaceholderRange(offset=2, length=3),
19
+ )
20
+
21
+ positions, delta = Qwen3ASRForConditionalGeneration.get_mrope_input_positions(
22
+ None, input_tokens, [feature]
23
+ )
24
+
25
+ expected = torch.arange(len(input_tokens), dtype=torch.long).expand(3, -1)
26
+ torch.testing.assert_close(positions, expected)
27
+ assert delta == 0
vllm/model_executor/models/qwen3_asr.py
+4
-7
@@ -575,13 +575,10 @@ class Qwen3ASRForConditionalGeneration(
575
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
576
offset = mm_feature.mm_position.offset
577
578
- # Get audio feature length from mm_feature data
579
- audio_feature_length = mm_feature.data["audio_feature_lengths"].data
580
- if isinstance(audio_feature_length, torch.Tensor):
581
- audio_feature_length = audio_feature_length.item()
582
- audio_len = _get_feat_extract_output_lengths(
583
- torch.tensor(audio_feature_length)
584
- ).item()
578
+ # The placeholder already contains one token per audio embedding.
579
+ # Use its length so M-RoPE can be reconstructed even when a covered
580
+ # prefix no longer carries the audio processor tensors.
581
+ audio_len = mm_feature.mm_position.length
582
583
# Text segment before audio (includes audio_start token)
584
text_len = offset - st