@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 140 lines 5.8 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """Regression tests for Qwen2.5-Omni and Qwen3-Omni audio-in-video processor
4 caching.
5
6 Tests the use_audio_in_video feature where audio is extracted from video and
7 processed together with video frames in an interleaved manner.
8
9 Regression test: when use_audio_in_video=True and the multimodal processor
10 cache is warm, the second request goes through MultiModalProcessorSenderCache
11 which sets mm_kwargs["video"] items to None on a cache hit. The processor
12 must still detect use_audio_in_video=True (via token-count heuristic) and
13 produce the same prompt_token_ids as the first (cache-miss) request.
14
15 Without the fix the cache-hit path left use_audio_in_video=False, causing
16 audio placeholder tokens to be inserted separately instead of being derived
17 from the interleaved video placeholders – yielding a different (wrong) token
18 sequence on every subsequent request for the same video.
19 """
20
21 import numpy as np
22 import pytest
23
24 from vllm.multimodal import MULTIMODAL_REGISTRY
25 from vllm.multimodal.cache import LruKeyReplicatedSenderCache
26
27 from ....multimodal.utils import random_audio, random_video
28 from ...utils import build_model_context
29
30 MODELS = [
31 "Qwen/Qwen2.5-Omni-3B",
32 "Qwen/Qwen3-Omni-30B-A3B-Instruct",
33 ]
34
35
36 def create_mm_data(num_videos: int) -> dict[str, list]:
37 # Small video (8 frames, 64×64) and ~0.5 s of audio at 16 kHz so the test
38 # stays fast even without a GPU.
39 mm_data = dict[str, list](video=[], audio=[])
40 for i in range(num_videos):
41 rng = np.random.RandomState(i)
42 video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
43 audio, sr = random_audio(rng, min_len=8000, max_len=8001, sr=16000)
44 mm_data["video"].append(video)
45 mm_data["audio"].append((audio, sr))
46 return mm_data
47
48
49 @pytest.mark.parametrize("model_id", MODELS)
50 @pytest.mark.parametrize("num_videos", [1, 2])
51 def test_audio_in_video_cache_correctness(model_id: str, num_videos: int) -> None:
52 """Regression test for https://github.com/vllm-project/vllm/pull/36800.
53
54 MultiModalProcessorSenderCache.get_and_update_item returns (None, updates)
55 on a cache hit, so mm_kwargs["video"] items become None on the second call.
56 The Qwen processor override of _maybe_apply_prompt_updates must detect
57 use_audio_in_video=True via token-count heuristics and re-derive the audio
58 placeholders correctly.
59 """
60 ctx = build_model_context(
61 model_id,
62 limit_mm_per_prompt={"audio": num_videos, "image": 0, "video": num_videos},
63 mm_processor_cache_gb=1,
64 )
65
66 processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
67
68 # Sender cache: on a cache hit returns (None, prompt_updates) for each
69 # item, setting mm_kwargs["video"] = [None] – the exact condition that
70 # triggered the original bug.
71 sender_cache = LruKeyReplicatedSenderCache(ctx.model_config)
72
73 video_token_id = processor.info.get_hf_config().video_token_id
74
75 mm_data = create_mm_data(num_videos)
76 hf_processor_mm_kwargs = {"use_audio_in_video": True}
77
78 def run(processor, cache):
79 return processor(
80 [video_token_id] * num_videos,
81 mm_items=processor.info.parse_mm_data(mm_data),
82 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
83 cache=cache,
84 )["prompt_token_ids"]
85
86 baseline_ids = run(processor, cache=None)
87
88 # First call on the sender-cache processor: cache miss.
89 # mm_kwargs["video"] items are real tensors; use_audio_in_video is
90 # detected normally from the item data.
91 first_ids = run(processor, cache=sender_cache)
92 assert first_ids == baseline_ids, (
93 "Cache-miss call produced different prompt_token_ids than baseline.\n"
94 f" baseline : {baseline_ids}\n"
95 f" cache-miss: {first_ids}"
96 )
97
98 # Second call on the sender-cache processor: cache hit.
99 # MultiModalProcessorSenderCache.get_and_update_item returns (None, …),
100 # so mm_kwargs["video"] = [None]. Before the fix, use_audio_in_video was
101 # not detected, yielding wrong token ids.
102 second_ids = run(processor, cache=sender_cache)
103 assert second_ids == baseline_ids, (
104 "Cache-hit call produced different prompt_token_ids than baseline.\n"
105 "This is the regression introduced when use_audio_in_video detection\n"
106 "fails for None mm_kwargs items on a cache hit.\n"
107 f" baseline : {baseline_ids}\n"
108 f" cache-hit: {second_ids}"
109 )
110
111
112 @pytest.mark.parametrize("model_id", MODELS)
113 def test_use_audio_in_video_without_audio_track(model_id: str) -> None:
114 """A video with no audio track, combined with `use_audio_in_video=True`,
115 must raise a clear `ValueError` naming the mismatch, rather than an
116 opaque `StopIteration`-derived error from the underlying HF processor.
117 """
118 ctx = build_model_context(
119 model_id,
120 limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
121 )
122 processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
123 video_token_id = processor.info.get_hf_config().video_token_id
124
125 rng = np.random.RandomState(0)
126 video = random_video(rng, min_frames=8, max_frames=9, min_wh=64, max_wh=65)
127
128 # No "audio" key at all in this request's data: this is the "video has
129 # no audio track" case. (The audio limit above is 1, not 0, so that
130 # `mm_counts["audio"]` is still declared for this request; a 0 limit
131 # would make vLLM omit the "audio" key from `mm_counts` entirely, which
132 # trips an unrelated assertion in `_apply_hf_processor_main`.)
133 mm_data = {"video": [video]}
134
135 with pytest.raises(ValueError, match="doesn't have audio track"):
136 processor(
137 [video_token_id],
138 mm_items=processor.info.parse_mm_data(mm_data),
139 hf_processor_mm_kwargs={"use_audio_in_video": True},
140 )