[Bugfix] Fix Qwen3-Omni crash on video with no audio track when use_audio_in_video=True (#48420)
Signed-off-by: RyanJHamby <ryanhamby22@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Ryan Hamby committed
Aug 3, 2026 at 19:22 UTC
6a9109d865d8abbfe70777ea52520425f1be6d3f
3 files changed
+57
-1
tests/models/multimodal/processing/test_audio_in_video.py
+32
@@ -113,3 +113,35 @@ def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> Non
113
f" baseline : {baseline_ids}\n"
114
f" cache-hit: {second_ids}"
115
)
116
+
117
+
118
+@pytest.mark.parametrize("model_id", MODELS)
119
+def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
120
+ """
121
+ A video with no audio track, combined with `use_audio_in_video=True`,
122
+ must raise a clear `ValueError` naming the mismatch, rather than an
123
+ opaque `StopIteration`-derived error from the underlying HF processor.
124
+ """
125
+ ctx = build_model_context(
126
+ model_id,
127
+ limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
128
+ )
129
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config, cache=None)
130
+ video_token_id = processor.info.get_hf_config().video_token_id
131
+
132
+ rng = np.random.RandomState(0)
133
+ video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
134
+
135
+ # No "audio" key at all in this request's data: this is the "video has
136
+ # no audio track" case. (The audio limit above is 1, not 0, so that
137
+ # `mm_counts["audio"]` is still declared for this request; a 0 limit
138
+ # would make vLLM omit the "audio" key from `mm_counts` entirely, which
139
+ # trips an unrelated assertion in `_apply_hf_processor_mm_only`.)
140
+ mm_data = {"video": [video]}
141
+
142
+ with pytest.raises(ValueError, match="doesn't have audio track"):
143
+ processor(
144
+ [video_token_id],
145
+ mm_items=processor.info.parse_mm_data(mm_data),
146
+ hf_processor_mm_kwargs={"use_audio_in_video": True},
147
+ )
vllm/model_executor/models/qwen2_5_omni_thinker.py
+18
-1
@@ -508,6 +508,20 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
508
mm_kwargs = dict(
509
**mm_kwargs,
510
)
511
+ elif (
512
+ mm_kwargs.get("use_audio_in_video")
513
+ and "audio" not in mm_data
514
+ and mm_data.get("videos")
515
+ ):
516
+ # A subclass (e.g. Qwen3-Omni) may have already popped "audios"
517
+ # and populated mm_data["audio"] itself before delegating here,
518
+ # in which case this isn't a genuine "no audio" case. Likewise,
519
+ # mm_data can be empty on a multimodal-processor-cache hit, where
520
+ # there's nothing to (re-)process this call and the real result
521
+ # comes from the cache — not a genuine "no audio" case either.
522
+ raise ValueError(
523
+ "Video doesn't have audio track with `audio_in_video=True`"
524
+ )
525
526
merged = self.info.ctx.get_merged_mm_kwargs(mm_kwargs)
527
if mm_data.get("videos") and (
@@ -911,7 +925,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
925
926
use_audio_in_video = hf_processor_mm_kwargs.get("use_audio_in_video", False)
927
if use_audio_in_video and "video" in mm_counts:
914
- assert "audio" in mm_counts
928
+ if mm_counts.get("audio", 0) < mm_counts["video"]:
929
+ raise ValueError(
930
+ "Video doesn't have audio track with `audio_in_video=True`"
931
+ )
932
mm_counts["audio"] -= mm_counts["video"]
933
934
_, mm_processed_data, _ = self._apply_hf_processor_text_mm(
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+7
@@ -1224,6 +1224,13 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1224
tok_kwargs = dict(tok_kwargs)
1225
mm_kwargs["audio_kwargs"] = dict(mm_kwargs.get("audio_kwargs") or {})
1226
mm_kwargs["text_kwargs"] = dict(mm_kwargs.get("text_kwargs") or {})
1227
+ elif mm_kwargs.get("use_audio_in_video") and mm_data.get("videos"):
1228
+ # mm_data can be empty on a multimodal-processor-cache hit, where
1229
+ # there's nothing to (re-)process this call and the real result
1230
+ # comes from the cache — not a genuine "no audio" case.
1231
+ raise ValueError(
1232
+ "Video doesn't have audio track with `audio_in_video=True`"
1233
+ )
1234
1235
hf_inputs = super()._call_hf_processor(
1236
prompt=prompt,