A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 15:15 UTC
[Bugfix][Multimodal] PyAV video backend returns keyframes labeled as targets (#42586)
Signed-off-by: Ranran <hzz5361@psu.edu>
Ranran committed
May 14, 2026 at 11:56 UTC
f3d5360591462f69146c24cde013435f0c0b1b9c
3 files changed
+100
-6
tests/multimodal/test_video.py
+44
-1
@@ -13,7 +13,7 @@ from vllm.multimodal.video import (
13
VideoLoader,
14
)
15
16
-from .utils import create_video_from_image
16
+from .utils import create_long_gop_video, create_video_from_image
17
18
pytestmark = pytest.mark.cpu_test
19
@@ -364,6 +364,49 @@ def test_pyav_dynamic_backend_loads_frames(
364
assert metadata["video_backend"] == "pyav_dynamic"
365
366
367
+def test_pyav_backend_returns_target_frames_not_keyframes():
368
+ """Regression test: PyAV must decode forward past the seek keyframe.
369
+
370
+ container.seek() snaps backward to the nearest keyframe. With a long GOP
371
+ (here: one keyframe at frame 0), a decoder that does not advance forward
372
+ to the target PTS collapses every sampled slot onto the keyframe. This
373
+ test encodes a per-frame marker on the green channel and verifies the
374
+ returned frames are distinct, ordered, and match the requested indices.
375
+ """
376
+ num_frames = 50
377
+ num_sampled = 4
378
+ height, width = 64, 64
379
+
380
+ video_bytes = create_long_gop_video(
381
+ num_frames=num_frames, width=width, height=height
382
+ )
383
+
384
+ loader = VIDEO_LOADER_REGISTRY.load("opencv")
385
+ frames, metadata = loader.load_bytes(
386
+ video_bytes, num_frames=num_sampled, backend="pyav"
387
+ )
388
+ assert frames.shape == (num_sampled, height, width, 3)
389
+
390
+ requested = list(metadata["frames_indices"])
391
+ assert len(requested) == num_sampled
392
+
393
+ actual = [int(f[height // 2, width // 2, 1]) for f in frames]
394
+
395
+ assert len(set(actual)) == num_sampled, (
396
+ f"PyAV returned only {len(set(actual))} distinct frames for "
397
+ f"{num_sampled} requested indices: markers={actual}, "
398
+ f"requested={requested}. Keyframe-snap regression."
399
+ )
400
+
401
+ assert actual == sorted(actual), f"Returned frames out of order: markers={actual}"
402
+
403
+ for marker, want_idx in zip(actual, requested):
404
+ assert abs(marker - want_idx) <= 10, (
405
+ f"Frame mismatch: requested index {want_idx}, "
406
+ f"got marker {marker} (tolerance ±10)"
407
+ )
408
+
409
+
410
@pytest.mark.parametrize(
411
"loader_key, kwargs, expected_num_frames",
412
[
tests/multimodal/utils.py
+37
@@ -66,6 +66,43 @@ def create_video_from_image(
66
return video_path
67
68
69
+def create_long_gop_video(
70
+ num_frames: int = 50,
71
+ fps: int = 30,
72
+ width: int = 64,
73
+ height: int = 64,
74
+) -> bytes:
75
+ """Encode an H.264 clip with one keyframe and green-channel = frame index.
76
+
77
+ The marker lets a test recover which frame the decoder actually returned,
78
+ independent of any metadata label.
79
+ """
80
+ import io
81
+
82
+ import av
83
+
84
+ buf = io.BytesIO()
85
+ with av.open(buf, mode="w", format="mp4") as container:
86
+ stream = container.add_stream("h264", rate=fps)
87
+ stream.width = width
88
+ stream.height = height
89
+ stream.pix_fmt = "yuv420p"
90
+ stream.codec_context.gop_size = num_frames
91
+ stream.codec_context.max_b_frames = 0
92
+ stream.codec_context.options = {
93
+ "x264-params": (f"scenecut=0:keyint={num_frames}:min-keyint={num_frames}")
94
+ }
95
+ for i in range(num_frames):
96
+ img = np.zeros((height, width, 3), dtype=np.uint8)
97
+ img[:, :, 1] = i
98
+ frame = av.VideoFrame.from_ndarray(img, format="rgb24")
99
+ for packet in stream.encode(frame):
100
+ container.mux(packet)
101
+ for packet in stream.encode():
102
+ container.mux(packet)
103
+ return buf.getvalue()
104
+
105
+
106
def cosine_similarity(A: npt.NDArray, B: npt.NDArray, axis: int = -1) -> npt.NDArray:
107
"""Compute cosine similarity between two vectors."""
108
return np.sum(A * B, axis=axis) / (
vllm/multimodal/video.py
+19
-5
@@ -390,7 +390,7 @@ class PyAVVideoBackendMixin:
390
fps: float,
391
duration: float,
392
) -> tuple[npt.NDArray, list[int]]:
393
- """Decode target frames via per-frame seek + keyframe decode."""
393
+ """Decode target frames via per-frame seek + forward decode to PTS."""
394
stream = container.streams.video[0]
395
# SLICE parallelizes within a single frame without the
396
# one-frame-per-thread latency penalty of FRAME threading.
@@ -402,14 +402,28 @@ class PyAVVideoBackendMixin:
402
frame_interval = 1.0 / fps if fps > 0 else 0.1
403
max_ts = max(0.0, duration - frame_interval) if duration > 0 else float("inf")
404
405
+ decoder = None
406
+ last_pts = None
407
for idx in frame_indices:
408
ts = min(idx / fps, max_ts) if fps > 0 else 0.0
409
pts = int(ts / time_base)
408
- container.seek(pts, stream=stream)
409
- frame = next(container.decode(video=0), None)
410
- if frame is not None:
411
- frames_list.append(frame.to_ndarray(format="rgb24"))
410
+ # seek() snaps backward to a keyframe; reuse the running decoder
411
+ # while targets advance monotonically to avoid re-decoding the
412
+ # GOP prefix once per requested frame.
413
+ if decoder is None or last_pts is None or pts <= last_pts:
414
+ container.seek(pts, stream=stream)
415
+ decoder = container.decode(video=0)
416
+ chosen = None
417
+ for frame in decoder:
418
+ if frame.pts is not None and frame.pts >= pts:
419
+ chosen = frame
420
+ last_pts = frame.pts
421
+ break
422
+ if chosen is not None:
423
+ frames_list.append(chosen.to_ndarray(format="rgb24"))
424
valid_indices.append(idx)
425
+ else:
426
+ decoder = None
427
428
if not frames_list:
429
return np.empty((0,), dtype=np.uint8), valid_indices