A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 21:45 UTC
[Bugfix] Fix Qwen3-VL timestamp mismatch when using num_frames without fps (#36136)
Signed-off-by: OiPunk <codingpunk@gmail.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Weiguang Li committed
Mar 11, 2026 at 18:13 UTC
724759684cd97a7a8625513c9a61bf95eaa396f1
2 files changed
+116
-4
tests/models/multimodal/processing/test_qwen3_vl.py
new
+94
@@ -0,0 +1,94 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Regression tests for Qwen3-VL processor.
4
+
5
+Covers the fix for num_frames-based timestamp calculation
6
+(issue vllm-project/vllm#35909).
7
+"""
8
+
9
+from typing import Any
10
+
11
+import numpy as np
12
+import pytest
13
+
14
+from vllm.multimodal import MULTIMODAL_REGISTRY
15
+
16
+from ...utils import build_model_context
17
+
18
+MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
19
+
20
+
21
+def _build_video_mm_data(
22
+ num_frames: int,
23
+ width: int = 128,
24
+ height: int = 128,
25
+ original_fps: float = 30.0,
26
+) -> dict[str, Any]:
27
+ """Create synthetic video data with metadata indicating that
28
+ HF processor should re-sample frames (do_sample_frames=True).
29
+
30
+ ``total_num_frames`` is set equal to the ndarray frame count so
31
+ that HF's ``sample_frames`` indices stay within bounds of the
32
+ actual tensor that is passed."""
33
+ video = np.zeros((num_frames, height, width, 3), dtype=np.uint8)
34
+ metadata = {
35
+ "fps": original_fps,
36
+ "duration": num_frames / original_fps,
37
+ "total_num_frames": num_frames,
38
+ "frames_indices": list(range(num_frames)),
39
+ "video_backend": "opencv",
40
+ "do_sample_frames": True,
41
+ }
42
+ return {"video": [(video, metadata)]}
43
+
44
+
45
+@pytest.mark.parametrize("model_id", [MODEL_ID])
46
+@pytest.mark.parametrize(
47
+ "num_frames",
48
+ [8, 16],
49
+)
50
+def test_processor_num_frames_timestamp(
51
+ model_id: str,
52
+ num_frames: int,
53
+) -> None:
54
+ """Regression test: using ``num_frames`` (without ``fps``) must not
55
+ cause a timestamp / token-count mismatch.
56
+
57
+ Before the fix, ``_get_video_second_idx`` ignored the explicit
58
+ ``num_frames`` and fell back to an fps-based calculation, which
59
+ produced a different number of timestamp entries and ultimately led
60
+ to shape mismatches in downstream token construction.
61
+
62
+ We deliberately choose ``num_frames`` values (8, 16) that differ
63
+ from what the default fps-based path would compute (which clamps
64
+ to ``min_frames=4`` for a short video at 30 fps), so this test
65
+ would fail without the fix.
66
+ """
67
+ ctx = build_model_context(
68
+ model_id,
69
+ limit_mm_per_prompt={"image": 0, "video": 1},
70
+ )
71
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
72
+
73
+ prompt = "<|vision_start|><|video_pad|><|vision_end|>"
74
+ mm_data = _build_video_mm_data(num_frames=num_frames)
75
+
76
+ # Process with explicit num_frames (no fps) -- this is the path
77
+ # that was broken before the fix.
78
+ hf_mm_kwargs: dict[str, Any] = {"num_frames": num_frames}
79
+ processed = processor(
80
+ prompt,
81
+ mm_items=processor.info.parse_mm_data(mm_data),
82
+ hf_processor_mm_kwargs=hf_mm_kwargs,
83
+ )
84
+
85
+ # Basic sanity: the processor must produce video tokens.
86
+ token_ids = processed["prompt_token_ids"]
87
+ assert len(token_ids) > 0, "Processor produced empty token list"
88
+
89
+ # Verify that video placeholders were actually inserted.
90
+ assert "mm_placeholders" in processed
91
+ video_phs = processed["mm_placeholders"].get("video", [])
92
+ assert len(video_phs) == 1, (
93
+ f"Expected exactly 1 video placeholder, got {len(video_phs)}"
94
+ )
vllm/model_executor/models/qwen3_vl.py
+22
-4
@@ -768,6 +768,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
768
metadata: dict[str, Any],
769
do_sample_frames: bool | None = None,
770
sampled_fps: float | None = None,
771
+ sampled_num_frames: int | None = None,
772
) -> list[int]:
773
video_processor = self.get_video_processor()
774
merge_size = video_processor.merge_size
@@ -782,11 +783,20 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
783
# video loader), we need to re-calculate the indices from original
784
# metadata.
785
if do_sample_frames:
785
- # here video_fps is the fps of the sampled video, and
786
- # metadata["fps"] refers to the fps of the original video.
787
- sampled_fps = sampled_fps if sampled_fps else video_processor.fps
786
total_num_frames = metadata["total_num_frames"]
789
- num_frames = int(total_num_frames / metadata["fps"] * sampled_fps)
787
+
788
+ # When num_frames is explicitly provided, use it directly
789
+ # instead of computing from fps. This mirrors the behavior of
790
+ # HF's Qwen3VLVideoProcessor.sample_frames where num_frames
791
+ # and fps are mutually exclusive.
792
+ if sampled_num_frames is not None:
793
+ num_frames = sampled_num_frames
794
+ else:
795
+ # here video_fps is the fps of the sampled video, and
796
+ # metadata["fps"] refers to the fps of the original video.
797
+ sampled_fps = sampled_fps if sampled_fps else video_processor.fps
798
+ num_frames = int(total_num_frames / metadata["fps"] * sampled_fps)
799
+
800
num_frames = min(
801
min(
802
max(num_frames, video_processor.min_frames),
@@ -987,6 +997,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
997
metadata=metadata,
998
do_sample_frames=video_mm_kwargs["do_sample_frames"],
999
sampled_fps=video_mm_kwargs.get("fps"),
1000
+ sampled_num_frames=video_mm_kwargs.get("num_frames"),
1001
)
1002
timestamps_per_video.append(timestamps)
1003
@@ -994,6 +1005,13 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
1005
video_mm_data["videos"] = [[video_array]]
1006
video_mm_data["video_metadata"] = [[metadata]]
1007
1008
+ # When num_frames is specified, explicitly set fps=None
1009
+ # to prevent HF's BaseVideoProcessor.preprocess() from
1010
+ # filling in the class default (fps=2) via setdefault(),
1011
+ # which would conflict with num_frames (mutually exclusive).
1012
+ if "num_frames" in video_mm_kwargs and "fps" not in video_mm_kwargs:
1013
+ video_mm_kwargs["fps"] = None
1014
+
1015
video_outputs = super()._call_hf_processor(
1016
prompt="<|vision_start|><|video_pad|><|vision_end|>",
1017
mm_data=video_mm_data,