[Bugfix] Video loading: sample over presentable frames, not header sample count (MP4 edit-list trims) (#48608)
Signed-off-by: AmitMY <amitmoryossef@gmail.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Amit Moryossef committed
Aug 20, 2026 at 06:23 UTC
9b2aef512440f024fc98cd0ac954cd452613ad5b
4 files changed
+117
-1
tests/multimodal/test_video.py
+51
-1
@@ -40,7 +40,11 @@ from vllm.multimodal.video_decoders.pynvvideocodec import (
40
from vllm.platforms import current_platform
41
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
42
43
-from .utils import create_long_gop_video, create_video_from_image
43
+from .utils import (
44
+ create_edit_list_trimmed_video,
45
+ create_long_gop_video,
46
+ create_video_from_image,
47
+)
48
49
pytestmark = pytest.mark.cpu_test
50
@@ -862,6 +866,44 @@ def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
866
)
867
868
869
+def test_video_backend_handles_edit_list_trimmed_video(
870
+ monkeypatch: pytest.MonkeyPatch,
871
+):
872
+ """
873
+ An mp4 edit list (e.g. from a lossless ``ffmpeg -ss ... -c copy`` cut)
874
+ hides the decode lead-in: the header still counts every physical sample
875
+ while sequential decode only yields the visible frames. Sampling over the
876
+ header count used to collapse such videos to the few indices below the
877
+ visible count (a single frame for the Qwen loaders) — the loader must
878
+ resample over the true stream length instead.
879
+ """
880
+ with monkeypatch.context() as m:
881
+ m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
882
+
883
+ video_data, num_visible = create_edit_list_trimmed_video(
884
+ num_frames=90, trim_start_frame=60
885
+ )
886
+
887
+ loader = VIDEO_LOADER_REGISTRY.load("opencv")
888
+ for backend in ["opencv", "pyav"]:
889
+ frames, metadata = loader.load_bytes(
890
+ video_data, num_frames=-1, backend=backend
891
+ )
892
+ assert metadata["total_num_frames"] == num_visible, backend
893
+ assert frames.shape[0] == num_visible, backend
894
+ assert len(metadata["frames_indices"]) == num_visible, backend
895
+ # The green channel encodes the source frame index: the visible
896
+ # frames must be the trailing ones (60..89), not the lead-in.
897
+ mean_green = frames[..., 1].reshape(frames.shape[0], -1).mean(axis=1)
898
+ assert abs(mean_green[0] - 60) <= 5, backend
899
+ assert abs(mean_green[-1] - 89) <= 5, backend
900
+
901
+ # The Qwen samplers used to degenerate to a single frame here.
902
+ qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(video_data)
903
+ assert qwen_metadata["total_num_frames"] == num_visible
904
+ assert qwen_frames.shape[0] >= 4
905
+
906
+
907
# ============================================================================
908
# Frame Recovery Tests
909
# ============================================================================
@@ -912,6 +954,14 @@ def test_video_recovery_simulated_failures(monkeypatch: pytest.MonkeyPatch):
954
def get(self, prop):
955
return self._cap.get(prop)
956
957
+ def set(self, prop, value):
958
+ # get_video_metadata probes the stream end with a seek;
959
+ # keep the simulated frame counter aligned with rewinds.
960
+ result = self._cap.set(prop, value)
961
+ if prop == cv2.CAP_PROP_POS_FRAMES:
962
+ self._current_frame = int(value) - 1
963
+ return result
964
+
965
def isOpened(self):
966
return self._cap.isOpened()
967
tests/multimodal/utils.py
+37
@@ -103,6 +103,43 @@ def create_long_gop_video(
103
return buf.getvalue()
104
105
106
+def create_edit_list_trimmed_video(
107
+ num_frames: int = 90,
108
+ trim_start_frame: int = 60,
109
+ fps: int = 30,
110
+) -> tuple[bytes, int]:
111
+ """Stream-copy-cut a long-GOP clip so an mp4 edit list hides the lead-in.
112
+
113
+ Remuxes the tail of a single-keyframe clip without re-encoding, rebasing
114
+ timestamps at ``trim_start_frame``: the lead-in packets are still needed
115
+ to decode, so they stay in the file at negative pts and the muxer records
116
+ an edit list. The header sample count stays ``num_frames`` while only the
117
+ trailing frames are presentable — the shape produced by lossless trims
118
+ (e.g. ``ffmpeg -ss ... -c copy``). Returns (video_bytes, visible_frames).
119
+ """
120
+ import io
121
+
122
+ import av
123
+
124
+ src = create_long_gop_video(num_frames=num_frames, fps=fps)
125
+ buf = io.BytesIO()
126
+ with (
127
+ av.open(io.BytesIO(src)) as source,
128
+ av.open(buf, mode="w", format="mp4") as out,
129
+ ):
130
+ in_stream = source.streams.video[0]
131
+ out_stream = out.add_stream_from_template(in_stream)
132
+ start_pts = round(trim_start_frame / fps / in_stream.time_base)
133
+ for packet in source.demux(in_stream):
134
+ if packet.pts is None or packet.dts is None:
135
+ continue
136
+ packet.pts -= start_pts
137
+ packet.dts -= start_pts
138
+ packet.stream = out_stream
139
+ out.mux(packet)
140
+ return buf.getvalue(), num_frames - trim_start_frame
141
+
142
+
143
def cosine_similarity(A: npt.NDArray, B: npt.NDArray, axis: int = -1) -> npt.NDArray:
144
"""Compute cosine similarity between two vectors."""
145
return np.sum(A * B, axis=axis) / (
vllm/multimodal/video_decoders/opencv.py
+21
@@ -78,6 +78,27 @@ class OpenCVVideoBackendMixin:
78
def get_video_metadata(cap: "cv2.VideoCapture") -> VideoSourceMetadata:
79
total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
80
original_fps = cap.get(cv2.CAP_PROP_FPS)
81
+ # CAP_PROP_FRAME_COUNT counts every physical sample in the container,
82
+ # overstating the presentable frames when an mp4 edit list hides the
83
+ # decode lead-in (e.g. a lossless `ffmpeg -ss ... -c copy` trim).
84
+ # Cross-check with an end-of-stream seek: the FFMPEG backend reports
85
+ # the resulting position against the edit-list-aware timeline, which
86
+ # exposes the true frame count without decoding anything. The seek
87
+ # may land mid-stream on corrupt files, so only trust it when the
88
+ # stream really ends there. A one-frame tolerance keeps mere header
89
+ # miscounts from being "corrected".
90
+ if total_frames_num > 0 and cap.set(cv2.CAP_PROP_POS_AVI_RATIO, 1.0):
91
+ visible_frames = int(cap.get(cv2.CAP_PROP_POS_FRAMES))
92
+ at_stream_end = not cap.grab()
93
+ cap.set(cv2.CAP_PROP_POS_FRAMES, 0)
94
+ if at_stream_end and 0 < visible_frames < total_frames_num - 1:
95
+ logger.warning(
96
+ "Video header claims %d frames but only %d are "
97
+ "presentable; sampling over the true frame count.",
98
+ total_frames_num,
99
+ visible_frames,
100
+ )
101
+ total_frames_num = visible_frames
102
duration = total_frames_num / original_fps if original_fps > 0 else 0
103
return VideoSourceMetadata(
104
total_frames_num=total_frames_num,
vllm/multimodal/video_decoders/pyav.py
+8
@@ -62,6 +62,14 @@ class PyAVVideoBackendMixin:
62
duration = float(stream.duration * stream.time_base) if stream.duration else 0.0
63
if total_frames == 0 and duration > 0 and fps > 0:
64
total_frames = int(duration * fps)
65
+ elif duration > 0 and fps > 0 and total_frames > round(duration * fps) + 1:
66
+ # The header sample count can exceed the frames the presentation
67
+ # timeline actually holds — e.g. an mp4 edit list hides the decode
68
+ # lead-in of a stream-copied cut, but ``stream.frames`` still
69
+ # counts the hidden samples. Sampling indices past the visible
70
+ # range would silently collapse onto the last visible frame, so
71
+ # trust the (edit-list-aware) duration instead.
72
+ total_frames = round(duration * fps)
73
return VideoSourceMetadata(total_frames, fps, duration)
74
75
@staticmethod