A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 14:00 UTC
[Bugfix] Fix check_interleaved_audio_video false positive for batched non-interleaved requests (#35487)
Signed-off-by: linyueqian <linyueqian@outlook.com> Co-authored-by: Roger Wang <hey@rogerw.io>
Yueqian Lin committed
Feb 27, 2026 at 09:48 UTC
e8249378e414d76387a027a6ffb5bde8b9aff765
2 files changed
+51
-4
tests/models/multimodal/processing/test_qwen2_5_omni_embed.py
+26
@@ -116,6 +116,32 @@ class TestCheckInterleavedAudioVideo:
116
is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
117
)
118
119
+ def test_batched_non_interleaved_no_false_positive(self):
120
+ """
121
+ Regression test for https://github.com/vllm-project/vllm/issues/35394.
122
+
123
+ 5 identical non-interleaved mixed-modality requests batched together:
124
+ each has [audio][image][video] in separate blocks with text between them.
125
+ Across the batch, audio from request N falls between video blocks of
126
+ request N and request N+1, causing the global ranges to overlap.
127
+ check_interleaved_audio_video must return False (not a false positive).
128
+ """
129
+ # Build one request: [text][audio*5][text][image*4][text][video*6][text]
130
+ single_ids, _ = make_token_seq(5, 4, 6)
131
+ # Batch 5 identical requests (separated by text tokens to simulate padding)
132
+ sep = torch.tensor([TEXT_TOKEN_ID] * 3)
133
+ batched_ids = torch.cat([single_ids, sep] * 5)
134
+ is_multimodal = (
135
+ (batched_ids == AUDIO_TOKEN_ID)
136
+ | (batched_ids == IMAGE_TOKEN_ID)
137
+ | (batched_ids == VIDEO_TOKEN_ID)
138
+ )
139
+ is_video = is_multimodal & (batched_ids == VIDEO_TOKEN_ID)
140
+ is_audio = is_multimodal & (batched_ids == AUDIO_TOKEN_ID)
141
+ assert not check_interleaved_audio_video(
142
+ is_video, is_audio, is_video.sum().item(), is_audio.sum().item()
143
+ ), "Batched non-interleaved requests should not be detected as interleaved"
144
+
145
146
# ---------------------------------------------------------------------------
147
# Tests for embed_input_ids via a minimal mock
vllm/model_executor/models/qwen2_5_omni_thinker.py
+25
-4
@@ -122,8 +122,17 @@ def check_interleaved_audio_video(
122
"""
123
Check if video and audio positions are interleaved in the multimodal region.
124
125
- Returns:
126
- True if video and audio tokens are interleaved, False otherwise.
125
+ Returns True only for the use_audio_in_video=True case, where video and
126
+ audio tokens alternate within a single contiguous region with no gaps.
127
+
128
+ A simple range-overlap check produces false positives when multiple
129
+ non-interleaved requests are batched together: audio tokens from request N
130
+ fall between video tokens from request N and request N+1, making the
131
+ global ranges overlap even though each individual request is non-interleaved.
132
+
133
+ To distinguish true interleaving from this batching artefact we require
134
+ that every position in the combined [first_VA, last_VA] range is occupied
135
+ by either a video or an audio token (no text/image gaps).
136
"""
137
if num_video == 0 or num_audio == 0:
138
return False
@@ -131,10 +140,22 @@ def check_interleaved_audio_video(
140
video_pos = is_video.nonzero(as_tuple=True)[0]
141
audio_pos = is_audio.nonzero(as_tuple=True)[0]
142
134
- return (
143
+ # Quick range-overlap pre-check (necessary but not sufficient).
144
+ if not (
145
video_pos[0].item() < audio_pos[-1].item()
146
and audio_pos[0].item() < video_pos[-1].item()
137
- )
147
+ ):
148
+ return False
149
+
150
+ # Density check: for true use_audio_in_video interleaving every position
151
+ # in the combined span is a video or audio token. Batched non-interleaved
152
+ # requests have text/image tokens between the per-request V and A blocks.
153
+ # combined_start/end encompass all V/A tokens, so num_video + num_audio
154
+ # equals the number of V/A tokens in range; compare directly to span size.
155
+ combined_start = min(video_pos[0].item(), audio_pos[0].item())
156
+ combined_end = max(video_pos[-1].item(), audio_pos[-1].item())
157
+ total_in_range = combined_end - combined_start + 1
158
+ return (num_video + num_audio) == total_in_range
159
160
161
def merge_interleaved_embeddings(