A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 17:00 UTC
[Bugfix] Fix Qwen2.5-Omni/Qwen3-Omni use_audio_in_video with multi-video inputs (#37147)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Mar 16, 2026 at 16:56 UTC
912fbe9555f9f2b5f402ba1a60e3d17828bc76b0
4 files changed
+117
-17
tests/entrypoints/openai/test_audio_in_video.py
+97
-2
@@ -18,10 +18,10 @@ MODEL_NAME = "Qwen/Qwen2.5-Omni-3B"
18
def server():
19
args = [
20
"--max-model-len",
21
- "8192",
21
+ "16384",
22
"--enforce-eager",
23
"--limit-mm-per-prompt",
24
- json.dumps({"audio": 1, "video": 1}),
24
+ json.dumps({"audio": 3, "video": 3}),
25
]
26
27
with RemoteOpenAIServer(
@@ -78,3 +78,98 @@ async def test_online_audio_in_video(
78
assert len(chat_completion.choices) == 1
79
choice = chat_completion.choices[0]
80
assert choice.finish_reason == "length"
81
+
82
+
83
+@pytest.mark.core_model
84
+@pytest.mark.asyncio
85
+async def test_online_audio_in_video_multi_videos(
86
+ client: openai.AsyncOpenAI, video_assets: VideoTestAssets
87
+):
88
+ """Test multi-video input with `audio_in_video=True`"""
89
+
90
+ # we don't use video_urls above because they missed audio stream.
91
+ video_path = video_assets[0].video_path
92
+ with open(video_path, "rb") as f:
93
+ video_base64 = base64.b64encode(f.read()).decode("utf-8")
94
+
95
+ messages = [
96
+ {
97
+ "role": "user",
98
+ "content": [
99
+ {"type": "text", "text": "What's in these two videos?"},
100
+ {
101
+ "type": "video_url",
102
+ "video_url": {"url": f"data:video/mp4;base64,{video_base64}"},
103
+ },
104
+ {
105
+ "type": "video_url",
106
+ "video_url": {"url": f"data:video/mp4;base64,{video_base64}"},
107
+ },
108
+ ],
109
+ }
110
+ ]
111
+
112
+ # multi-turn to test mm processor cache as well
113
+ for _ in range(2):
114
+ chat_completion = await client.chat.completions.create(
115
+ model=MODEL_NAME,
116
+ messages=messages,
117
+ max_tokens=16,
118
+ extra_body={
119
+ "mm_processor_kwargs": {
120
+ "use_audio_in_video": True,
121
+ }
122
+ },
123
+ )
124
+
125
+ assert len(chat_completion.choices) == 1
126
+ choice = chat_completion.choices[0]
127
+ assert choice.finish_reason == "length"
128
+
129
+
130
+@pytest.mark.core_model
131
+@pytest.mark.asyncio
132
+async def test_online_audio_in_video_interleaved(
133
+ client: openai.AsyncOpenAI, video_assets: VideoTestAssets
134
+):
135
+ """Test interleaved video/audio input with `audio_in_video=True`"""
136
+
137
+ # we don't use video_urls above because they missed audio stream.
138
+ video_path = video_assets[0].video_path
139
+ with open(video_path, "rb") as f:
140
+ video_base64 = base64.b64encode(f.read()).decode("utf-8")
141
+
142
+ messages = [
143
+ {
144
+ "role": "user",
145
+ "content": [
146
+ {"type": "text", "text": "What's in these two videos?"},
147
+ {
148
+ "type": "video_url",
149
+ "video_url": {"url": f"data:video/mp4;base64,{video_base64}"},
150
+ },
151
+ {
152
+ "type": "audio_url",
153
+ "audio_url": {"url": f"data:audio/mp4;base64,{video_base64}"},
154
+ },
155
+ {
156
+ "type": "video_url",
157
+ "video_url": {"url": f"data:video/mp4;base64,{video_base64}"},
158
+ },
159
+ ],
160
+ }
161
+ ]
162
+ with pytest.raises(
163
+ openai.BadRequestError,
164
+ match="use_audio_in_video requires equal number of audio and video items",
165
+ ):
166
+ await client.chat.completions.create(
167
+ model=MODEL_NAME,
168
+ messages=messages,
169
+ max_tokens=16,
170
+ extra_body={
171
+ "mm_processor_kwargs": {
172
+ "use_audio_in_video": True,
173
+ }
174
+ },
175
+ )
tests/models/multimodal/processing/test_audio_in_video.py
+18
-9
@@ -34,8 +34,22 @@ MODELS = [
34
]
35
36
37
+def create_mm_data(num_videos: int) -> dict[str, list]:
38
+ # Small video (8 frames, 64×64) and ~0.5 s of audio at 16 kHz so the test
39
+ # stays fast even without a GPU.
40
+ mm_data = dict[str, list](video=[], audio=[])
41
+ for i in range(num_videos):
42
+ rng = np.random.RandomState(i)
43
+ video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
44
+ audio, sr = random_audio(rng, min_len=8000, max_len=8001, sr=16000)
45
+ mm_data["video"].append(video)
46
+ mm_data["audio"].append((audio, sr))
47
+ return mm_data
48
+
49
+
50
@pytest.mark.parametrize("model_id", MODELS)
38
-def test_audio_in_video_cache_correctness(model_id: str) -> None:
51
+@pytest.mark.parametrize("num_videos", [1, 2])
52
+def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> None:
53
"""
54
Regression test for https://github.com/vllm-project/vllm/pull/36800
55
@@ -47,7 +61,7 @@ def test_audio_in_video_cache_correctness(model_id: str) -> None:
61
"""
62
ctx = build_model_context(
63
model_id,
50
- limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
64
+ limit_mm_per_prompt={"audio": num_videos, "image": 0, "video": num_videos},
65
mm_processor_cache_gb=1,
66
)
67
@@ -65,17 +79,12 @@ def test_audio_in_video_cache_correctness(model_id: str) -> None:
79
80
video_token_id = baseline_processor.info.get_hf_config().video_token_id
81
68
- rng = np.random.RandomState(0)
69
- # Small video (8 frames, 64×64) and ~0.5 s of audio at 16 kHz so the test
70
- # stays fast even without a GPU.
71
- video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
72
- audio, sr = random_audio(rng, min_len=8000, max_len=8001, sr=16000)
73
- mm_data = {"video": [video], "audio": [(audio, sr)]}
82
+ mm_data = create_mm_data(num_videos)
83
hf_processor_mm_kwargs = {"use_audio_in_video": True}
84
85
def run(processor):
86
return processor(
78
- [video_token_id],
87
+ [video_token_id] * num_videos,
88
mm_items=baseline_processor.info.parse_mm_data(mm_data),
89
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
90
)["prompt_token_ids"]
vllm/model_executor/models/qwen2_5_omni_thinker.py
+1
-3
@@ -774,9 +774,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
774
def get_replacement_qwen2_use_audio_in_video(item_idx: int):
775
nonlocal audio_in_video_item_idx
776
777
- audio_num_features = audio_output_lengths[
778
- audio_in_video_item_idx + item_idx
779
- ]
777
+ audio_num_features = audio_output_lengths[audio_in_video_item_idx]
778
video_grid_thw = out_mm_data["video_grid_thw"][item_idx]
779
780
audio_in_video_item_idx += 1
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+1
-3
@@ -1489,9 +1489,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1489
1490
def get_replacement_qwen2_use_audio_in_video(item_idx: int):
1491
nonlocal audio_in_video_item_idx
1492
- audio_num_features = audio_output_lengths[
1493
- audio_in_video_item_idx + item_idx
1494
- ]
1492
+ audio_num_features = audio_output_lengths[audio_in_video_item_idx]
1493
video_grid_thw = out_mm_data["video_grid_thw"][item_idx]
1494
1495
audio_in_video_item_idx += 1