A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 19:15 UTC
[Bugfix] Fix Qwen2.5-omni/Qwen3-omni mm_processor cache for audio_in_video request (#36800)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Mar 14, 2026 at 02:16 UTC
abf61aaa8ef2facaf82bc8fd3a9fb545ccf14b3d
3 files changed
+128
-12
tests/models/multimodal/processing/test_audio_in_video.py
new
+106
@@ -0,0 +1,106 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+Regression tests for Qwen2.5-Omni and Qwen3-Omni audio-in-video processor
5
+caching.
6
+
7
+Tests the use_audio_in_video feature where audio is extracted from video and
8
+processed together with video frames in an interleaved manner.
9
+
10
+Regression test: when use_audio_in_video=True and the multimodal processor
11
+cache is warm, the second request goes through MultiModalProcessorSenderCache
12
+which sets mm_kwargs["video"] items to None on a cache hit. The processor
13
+must still detect use_audio_in_video=True (via token-count heuristic) and
14
+produce the same prompt_token_ids as the first (cache-miss) request.
15
+
16
+Without the fix the cache-hit path left use_audio_in_video=False, causing
17
+audio placeholder tokens to be inserted separately instead of being derived
18
+from the interleaved video placeholders – yielding a different (wrong) token
19
+sequence on every subsequent request for the same video.
20
+"""
21
+
22
+import numpy as np
23
+import pytest
24
+
25
+from vllm.multimodal import MULTIMODAL_REGISTRY
26
+from vllm.multimodal.cache import MultiModalProcessorSenderCache
27
+
28
+from ....multimodal.utils import random_audio, random_video
29
+from ...utils import build_model_context
30
+
31
+MODELS = [
32
+ "Qwen/Qwen2.5-Omni-3B",
33
+ "Qwen/Qwen3-Omni-30B-A3B-Instruct",
34
+]
35
+
36
+
37
+@pytest.mark.parametrize("model_id", MODELS)
38
+def test_audio_in_video_cache_correctness(model_id: str) -> None:
39
+ """
40
+ Regression test for https://github.com/vllm-project/vllm/pull/36800
41
+
42
+ MultiModalProcessorSenderCache.get_and_update_item returns (None, updates)
43
+ on a cache hit, so mm_kwargs["video"] items become None on the second call.
44
+ The Qwen processor override of _maybe_apply_prompt_updates must detect
45
+ use_audio_in_video=True via token-count heuristics and re-derive the audio
46
+ placeholders correctly.
47
+ """
48
+ ctx = build_model_context(
49
+ model_id,
50
+ limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
51
+ mm_processor_cache_gb=1,
52
+ )
53
+
54
+ # Baseline: no cache, always processes from scratch.
55
+ baseline_processor = MULTIMODAL_REGISTRY.create_processor(
56
+ ctx.model_config, cache=None
57
+ )
58
+ # Sender cache: on a cache hit returns (None, prompt_updates) for each
59
+ # item, setting mm_kwargs["video"] = [None] – the exact condition that
60
+ # triggered the original bug.
61
+ sender_cache = MultiModalProcessorSenderCache(ctx.model_config)
62
+ cached_processor = MULTIMODAL_REGISTRY.create_processor(
63
+ ctx.model_config, cache=sender_cache
64
+ )
65
+
66
+ video_token_id = baseline_processor.info.get_hf_config().video_token_id
67
+
68
+ rng = np.random.RandomState(0)
69
+ # Small video (8 frames, 64×64) and ~0.5 s of audio at 16 kHz so the test
70
+ # stays fast even without a GPU.
71
+ video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
72
+ audio, sr = random_audio(rng, min_len=8000, max_len=8001, sr=16000)
73
+ mm_data = {"video": [video], "audio": [(audio, sr)]}
74
+ hf_processor_mm_kwargs = {"use_audio_in_video": True}
75
+
76
+ def run(processor):
77
+ return processor(
78
+ [video_token_id],
79
+ mm_items=baseline_processor.info.parse_mm_data(mm_data),
80
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
81
+ )["prompt_token_ids"]
82
+
83
+ baseline_ids = run(baseline_processor)
84
+
85
+ # First call on the sender-cache processor: cache miss.
86
+ # mm_kwargs["video"] items are real tensors; use_audio_in_video is
87
+ # detected normally from the item data.
88
+ first_ids = run(cached_processor)
89
+ assert first_ids == baseline_ids, (
90
+ "Cache-miss call produced different prompt_token_ids than baseline.\n"
91
+ f" baseline : {baseline_ids}\n"
92
+ f" cache-miss: {first_ids}"
93
+ )
94
+
95
+ # Second call on the sender-cache processor: cache hit.
96
+ # MultiModalProcessorSenderCache.get_and_update_item returns (None, …),
97
+ # so mm_kwargs["video"] = [None]. Before the fix, use_audio_in_video was
98
+ # not detected, yielding wrong token ids.
99
+ second_ids = run(cached_processor)
100
+ assert second_ids == baseline_ids, (
101
+ "Cache-hit call produced different prompt_token_ids than baseline.\n"
102
+ "This is the regression introduced when use_audio_in_video detection\n"
103
+ "fails for None mm_kwargs items on a cache hit.\n"
104
+ f" baseline : {baseline_ids}\n"
105
+ f" cache-hit: {second_ids}"
106
+ )
vllm/model_executor/models/qwen2_5_omni_thinker.py
+11
-12
@@ -80,8 +80,6 @@ from vllm.multimodal.parse import (
80
)
81
from vllm.multimodal.processing import (
82
BaseDummyInputsBuilder,
83
- ProcessorInputs,
84
- TimingContext,
83
)
84
from vllm.multimodal.processing.processor import (
85
BaseMultiModalProcessor,
@@ -609,6 +607,17 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
607
if use_audio_in_video_tensor.numel() > 0:
608
use_audio_in_video = bool(use_audio_in_video_tensor.item())
609
break
610
+ # for mutilmodality cache
611
+ if any(item is None for item in mm_kwargs["video"]):
612
+ video_token_id = self.info.get_hf_config().video_token_id
613
+ audio_token_id = self.info.get_hf_config().audio_token_id
614
+ video_audio_item_num = sum(
615
+ id in (video_token_id, audio_token_id) for id in prompt_ids
616
+ )
617
+ audio_updates_num = len(mm_prompt_updates.get("audio", []))
618
+ video_updates_num = len(mm_prompt_updates.get("video", []))
619
+ if video_audio_item_num != video_updates_num + audio_updates_num:
620
+ use_audio_in_video = True
621
622
if is_update_applied:
623
mm_placeholders = self._find_mm_placeholders(
@@ -815,16 +824,6 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
824
),
825
]
826
818
- def _cached_apply_hf_processor(
819
- self,
820
- inputs: ProcessorInputs,
821
- timing_ctx: TimingContext,
822
- ):
823
- mm_processor_kwargs = inputs.hf_processor_mm_kwargs
824
- if mm_processor_kwargs.get("use_audio_in_video", False):
825
- return self._apply_hf_processor(inputs, timing_ctx)
826
- return super()._cached_apply_hf_processor(inputs, timing_ctx)
827
-
827
def _apply_hf_processor_main(
828
self,
829
prompt: str | list[int],
vllm/model_executor/models/qwen3_omni_moe_thinker.py
+11
@@ -1326,6 +1326,17 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
1326
use_audio_in_video = True
1327
else:
1328
use_audio_in_video = False
1329
+ # for mutilmodality cache
1330
+ if any(item is None for item in mm_kwargs["video"]):
1331
+ video_token_id = self.info.get_hf_config().video_token_id
1332
+ audio_token_id = self.info.get_hf_config().audio_token_id
1333
+ video_audio_item_num = sum(
1334
+ id in (video_token_id, audio_token_id) for id in prompt_ids
1335
+ )
1336
+ audio_updates_num = len(mm_prompt_updates.get("audio", []))
1337
+ video_updates_num = len(mm_prompt_updates.get("video", []))
1338
+ if video_audio_item_num != video_updates_num + audio_updates_num:
1339
+ use_audio_in_video = True
1340
1341
# normal case with `use_audio_in_video=False`
1342
if is_update_applied: