A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 21:15 UTC
[Multimodal] Add PyAV video backend for concurrent video decoding (#39986)
Signed-off-by: Jaseel Muhammad <jaseel.muhammad@mbzuai.ac.ae> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Jaseel Muhammad committed
Apr 22, 2026 at 07:14 UTC
6f2c71be8ffd65648bbf99d571c5353ff5b77f24
4 files changed
+290
-118
tests/models/multimodal/processing/test_glm4_1v.py
+8
-4
@@ -6,7 +6,7 @@ import pytest
6
from vllm.assets.video import VideoAsset
7
from vllm.multimodal import MULTIMODAL_REGISTRY
8
from vllm.multimodal.inputs import batched_tensors_equal
9
-from vllm.multimodal.video import OpenCVDynamicVideoBackend, OpenCVVideoBackend
9
+from vllm.multimodal.video import DynamicVideoBackend, VideoBackend
10
11
from ...utils import build_model_context
12
@@ -70,9 +70,11 @@ def test_processor_override(
70
71
@pytest.mark.parametrize("model_id", ["zai-org/GLM-4.1V-9B-Thinking"])
72
@pytest.mark.parametrize("fps", [2])
73
+@pytest.mark.parametrize("backend", ["opencv", "pyav"])
74
def test_video_loader_consistency(
75
model_id: str,
76
fps: int,
77
+ backend: str,
78
):
79
"""
80
Ensure dynamic video loader (pre-sampled by loader) and normal video
@@ -93,9 +95,11 @@ def test_video_loader_consistency(
95
with open(video_path, "rb") as f:
96
video_bytes = f.read()
97
96
- static_video, static_metadata = OpenCVVideoBackend.load_bytes(video_bytes)
97
- dynamic_video, dynamic_metadata = OpenCVDynamicVideoBackend.load_bytes(
98
- video_bytes, fps=fps
98
+ static_video, static_metadata = VideoBackend.load_bytes(
99
+ video_bytes, backend=backend
100
+ )
101
+ dynamic_video, dynamic_metadata = DynamicVideoBackend.load_bytes(
102
+ video_bytes, fps=fps, backend=backend
103
)
104
105
# pre-sampled loader shouldn't read all frames
tests/multimodal/test_video.py
+104
-17
@@ -71,7 +71,9 @@ def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
71
video_data = f.read()
72
73
loader = VIDEO_LOADER_REGISTRY.load("opencv")
74
- frames, metadata = loader.load_bytes(video_data, num_frames=-1)
74
+ frames, metadata = loader.load_bytes(
75
+ video_data, num_frames=-1, backend="opencv"
76
+ )
77
78
# Verify metadata consistency:
79
# frames_indices must match actual loaded frames
@@ -158,12 +160,12 @@ def test_video_recovery_simulated_failures(monkeypatch: pytest.MonkeyPatch):
160
161
# Test WITHOUT recovery - should have fewer frames due to failures
162
frames_no_recovery, meta_no = loader.load_bytes(
161
- video_data, num_frames=8, frame_recovery=False
163
+ video_data, num_frames=8, frame_recovery=False, backend="opencv"
164
)
165
166
# Test WITH recovery - should recover using next valid frames
167
frames_with_recovery, meta_yes = loader.load_bytes(
166
- video_data, num_frames=8, frame_recovery=True
168
+ video_data, num_frames=8, frame_recovery=True, backend="opencv"
169
)
170
171
# With recovery should have MORE frames than without
@@ -214,12 +216,12 @@ def test_video_recovery_with_corrupted_file(monkeypatch: pytest.MonkeyPatch):
216
217
# Test without recovery - frame 17 will be skipped
218
frames_no_recovery, meta_no_recovery = loader.load_bytes(
217
- video_data, num_frames=8, frame_recovery=False
219
+ video_data, num_frames=8, frame_recovery=False, backend="opencv"
220
)
221
222
# Test with recovery - frame 18 should fill in for frame 17
223
frames_with_recovery, meta_with_recovery = loader.load_bytes(
222
- video_data, num_frames=8, frame_recovery=True
224
+ video_data, num_frames=8, frame_recovery=True, backend="opencv"
225
)
226
227
# Verify metadata consistency for both modes
@@ -271,12 +273,16 @@ def test_video_recovery_dynamic_backend(monkeypatch: pytest.MonkeyPatch):
273
274
# Test without recovery
275
frames_no_recovery, meta_no = loader.load_bytes(
274
- video_data, fps=2, max_duration=10, frame_recovery=False
276
+ video_data,
277
+ fps=2,
278
+ max_duration=10,
279
+ frame_recovery=False,
280
+ backend="opencv",
281
)
282
283
# Test with frame_recovery enabled
284
frames_with_recovery, meta_with = loader.load_bytes(
279
- video_data, fps=2, max_duration=10, frame_recovery=True
285
+ video_data, fps=2, max_duration=10, frame_recovery=True, backend="opencv"
286
)
287
288
# Verify basic properties
@@ -310,27 +316,81 @@ def dummy_video_path(tmp_path):
316
return video_path
317
318
319
+# ============================================================================
320
+# PyAV Backend Tests
321
+# ============================================================================
322
+
323
+
324
+def test_pyav_backend_loads_frames(dummy_video_path, monkeypatch: pytest.MonkeyPatch):
325
+ """Test that the pyav codec backend can load frames from a valid video."""
326
+ with monkeypatch.context() as m:
327
+ m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
328
+
329
+ with open(dummy_video_path, "rb") as f:
330
+ video_data = f.read()
331
+
332
+ loader = VIDEO_LOADER_REGISTRY.load("opencv")
333
+ frames, metadata = loader.load_bytes(video_data, num_frames=8, backend="pyav")
334
+
335
+ assert frames.ndim == 4
336
+ assert frames.shape[3] == 3 # RGB
337
+ assert frames.shape[0] == 8
338
+ assert frames.shape[0] == len(metadata["frames_indices"])
339
+ assert metadata["video_backend"] == "pyav"
340
+ assert "total_num_frames" in metadata
341
+ assert "fps" in metadata
342
+ assert "duration" in metadata
343
+
344
+
345
+def test_pyav_dynamic_backend_loads_frames(
346
+ dummy_video_path, monkeypatch: pytest.MonkeyPatch
347
+):
348
+ """Test that the pyav codec with dynamic sampling can load frames."""
349
+ with monkeypatch.context() as m:
350
+ m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic")
351
+
352
+ with open(dummy_video_path, "rb") as f:
353
+ video_data = f.read()
354
+
355
+ loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
356
+ frames, metadata = loader.load_bytes(
357
+ video_data, fps=2, max_duration=10, backend="pyav"
358
+ )
359
+
360
+ assert frames.ndim == 4
361
+ assert frames.shape[3] == 3 # RGB
362
+ assert frames.shape[0] > 0
363
+ assert frames.shape[0] == len(metadata["frames_indices"])
364
+ assert metadata["video_backend"] == "pyav_dynamic"
365
+
366
+
367
@pytest.mark.parametrize(
314
- "backend, kwargs, expected_num_frames",
368
+ "loader_key, kwargs, expected_num_frames",
369
[
316
- # opencv: num_frames directly controls count
317
- pytest.param("opencv", {"num_frames": 32}, 32, id="opencv-num_frames"),
318
- pytest.param("opencv", {"fps": 2}, 120, id="opencv-fps"),
370
+ # uniform sampling + opencv codec
371
+ pytest.param(
372
+ "opencv",
373
+ {"num_frames": 32, "backend": "opencv"},
374
+ 32,
375
+ id="opencv-num_frames",
376
+ ),
377
+ pytest.param("opencv", {"fps": 2, "backend": "opencv"}, 120, id="opencv-fps"),
378
pytest.param(
379
"opencv",
321
- {"num_frames": 500, "fps": 2},
380
+ {"num_frames": 500, "fps": 2, "backend": "opencv"},
381
120,
382
id="opencv-num_frames_wins_fps",
383
),
384
+ # dynamic sampling + opencv codec
385
pytest.param(
386
"opencv_dynamic",
327
- {"fps": 1, "max_duration": 60},
387
+ {"fps": 1, "max_duration": 60, "backend": "opencv"},
388
60,
389
id="opencv_dynamic-within_max_duration",
390
),
391
pytest.param(
392
"opencv_dynamic",
333
- {"fps": 2, "max_duration": 30},
393
+ {"fps": 2, "max_duration": 30, "backend": "opencv"},
394
60,
395
id="opencv_dynamic-exceeds_max_duration",
396
),
@@ -349,18 +409,45 @@ def dummy_video_path(tmp_path):
409
119,
410
id="molmo2-fps",
411
),
412
+ # uniform sampling + pyav codec (same frame counts as opencv)
413
+ pytest.param(
414
+ "opencv",
415
+ {"num_frames": 32, "backend": "pyav"},
416
+ 32,
417
+ id="pyav-num_frames",
418
+ ),
419
+ pytest.param("opencv", {"fps": 2, "backend": "pyav"}, 120, id="pyav-fps"),
420
+ pytest.param(
421
+ "opencv",
422
+ {"num_frames": 500, "fps": 2, "backend": "pyav"},
423
+ 120,
424
+ id="pyav-num_frames_wins_fps",
425
+ ),
426
+ # dynamic sampling + pyav codec
427
+ pytest.param(
428
+ "opencv_dynamic",
429
+ {"fps": 1, "max_duration": 60, "backend": "pyav"},
430
+ 60,
431
+ id="pyav_dynamic-within_max_duration",
432
+ ),
433
+ pytest.param(
434
+ "opencv_dynamic",
435
+ {"fps": 2, "max_duration": 30, "backend": "pyav"},
436
+ 60,
437
+ id="pyav_dynamic-exceeds_max_duration",
438
+ ),
439
],
440
)
441
def test_video_loader_frames_sampling(
442
dummy_video_path,
443
monkeypatch: pytest.MonkeyPatch,
357
- backend: str,
444
+ loader_key: str,
445
kwargs: dict,
446
expected_num_frames: int,
447
):
448
"""Test video loader frames sampling functionality."""
362
- monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", backend)
363
- loader = VIDEO_LOADER_REGISTRY.load(backend)
449
+ monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", loader_key)
450
+ loader = VIDEO_LOADER_REGISTRY.load(loader_key)
451
452
with open(dummy_video_path, "rb") as f:
453
long_video_bytes = f.read()
vllm/envs.py
+4
-3
@@ -829,9 +829,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
829
"VLLM_MAX_AUDIO_CLIP_FILESIZE_MB": lambda: int(
830
os.getenv("VLLM_MAX_AUDIO_CLIP_FILESIZE_MB", "25")
831
),
832
- # Backend for Video IO
833
- # - "opencv": Default backend that uses OpenCV stream buffered backend.
834
- # - "identity": Returns raw video bytes for model processor to handle.
832
+ # Backend for Video IO — selects the frame-sampling algorithm.
833
+ # - "opencv": uniform sampling.
834
+ # - "opencv_dynamic": duration-aware dynamic sampling.
835
+ # - "identity": returns raw video bytes for model processor to handle.
836
#
837
# Custom backend implementations can be registered
838
# via `@VIDEO_LOADER_REGISTRY.register("my_custom_video_loader")` and
vllm/multimodal/video.py
+174
-94
@@ -3,7 +3,7 @@
3
import math
4
from abc import abstractmethod
5
from io import BytesIO
6
-from typing import Any, NamedTuple, cast
6
+from typing import Any, ClassVar, Literal, NamedTuple, cast
7
8
import numpy as np
9
import numpy.typing as npt
@@ -19,6 +19,11 @@ except ImportError:
19
cv2 = PlaceholderModule("cv2")
20
vr = PlaceholderModule("cv2").placeholder_attr("videoio_registry")
21
22
+try:
23
+ import av
24
+except ImportError:
25
+ av = PlaceholderModule("av") # type: ignore[assignment]
26
+
27
28
logger = init_logger(__name__)
29
@@ -355,8 +360,75 @@ class OpenCVVideoBackendMixin:
360
return frames, valid_frame_indices
361
362
363
+class PyAVVideoBackendMixin:
364
+ """PyAV (in-process FFmpeg bindings) codec utilities.
365
+
366
+ Reads stream metadata and decodes target frames via per-frame
367
+ ``container.seek()``. The seek releases the GIL between frames and
368
+ scales with the number of sampled frames rather than the video
369
+ length, enabling concurrent decoding under serving load.
370
+ """
371
+
372
+ @staticmethod
373
+ def get_metadata(
374
+ container: "av.container.InputContainer",
375
+ ) -> VideoSourceMetadata:
376
+ if not container.streams.video:
377
+ raise ValueError("No video streams found in container")
378
+ stream = container.streams.video[0]
379
+ total_frames = stream.frames or 0
380
+ fps = float(stream.average_rate) if stream.average_rate else 0.0
381
+ duration = float(stream.duration * stream.time_base) if stream.duration else 0.0
382
+ if total_frames == 0 and duration > 0 and fps > 0:
383
+ total_frames = int(duration * fps)
384
+ return VideoSourceMetadata(total_frames, fps, duration)
385
+
386
+ @staticmethod
387
+ def decode_frames(
388
+ container: "av.container.InputContainer",
389
+ frame_indices: list[int],
390
+ fps: float,
391
+ duration: float,
392
+ ) -> tuple[npt.NDArray, list[int]]:
393
+ """Decode target frames via per-frame seek + keyframe decode."""
394
+ stream = container.streams.video[0]
395
+ # SLICE parallelizes within a single frame without the
396
+ # one-frame-per-thread latency penalty of FRAME threading.
397
+ stream.thread_type = "SLICE"
398
+ time_base = stream.time_base
399
+
400
+ frames_list: list[npt.NDArray] = []
401
+ valid_indices: list[int] = []
402
+ frame_interval = 1.0 / fps if fps > 0 else 0.1
403
+ max_ts = max(0.0, duration - frame_interval) if duration > 0 else float("inf")
404
+
405
+ for idx in frame_indices:
406
+ ts = min(idx / fps, max_ts) if fps > 0 else 0.0
407
+ pts = int(ts / time_base)
408
+ container.seek(pts, stream=stream)
409
+ frame = next(container.decode(video=0), None)
410
+ if frame is not None:
411
+ frames_list.append(frame.to_ndarray(format="rgb24"))
412
+ valid_indices.append(idx)
413
+
414
+ if not frames_list:
415
+ return np.empty((0,), dtype=np.uint8), valid_indices
416
+ return np.stack(frames_list), valid_indices
417
+
418
+
419
@VIDEO_LOADER_REGISTRY.register("opencv")
359
-class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
420
+class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
421
+ """Uniform-sampling video backend.
422
+
423
+ Samples ``num_frames`` uniformly across the video (or one frame every
424
+ ``1/fps`` seconds, whichever produces fewer frames). The decoding codec
425
+ is selected via the ``backend`` kwarg (``"opencv"`` or ``"pyav"``),
426
+ which can be passed through ``--media-io-kwargs``. Defaults to
427
+ ``"pyav"`` for concurrent decoding.
428
+ """
429
+
430
+ _sampling_suffix: ClassVar[str] = ""
431
+
432
@classmethod
433
def compute_frames_index_to_sample(
434
cls,
@@ -366,7 +438,6 @@ class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
438
) -> list[int]:
439
total_frames_num = source.total_frames_num
440
duration = source.duration
369
-
441
num_frames = target.num_frames
442
fps = target.fps
443
# resample video to target num_frames and fps
@@ -376,16 +447,18 @@ class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
447
num_frames_to_sample = min(num_frames, total_frames_num)
448
if fps > 0:
449
num_frames_to_sample = min(num_frames_to_sample, math.floor(duration * fps))
379
- num_frames_to_sample = max(1, num_frames_to_sample) # at least one sample
450
+ num_frames_to_sample = max(1, num_frames_to_sample)
451
452
if num_frames_to_sample == total_frames_num:
382
- frame_idx = list(range(0, num_frames_to_sample))
383
- else:
384
- uniform_sampled_frames = np.linspace(
385
- 0, total_frames_num - 1, num_frames_to_sample, dtype=int
386
- )
387
- frame_idx = uniform_sampled_frames.tolist()
388
- return frame_idx
453
+ return list(range(num_frames_to_sample))
454
+ return np.linspace(
455
+ 0, total_frames_num - 1, num_frames_to_sample, dtype=int
456
+ ).tolist()
457
+
458
+ @classmethod
459
+ def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:
460
+ """Sampling-algorithm-specific metadata adjustment hook."""
461
+ return source
462
463
@classmethod
464
def load_bytes(
@@ -395,55 +468,101 @@ class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
468
fps: int = -1,
469
max_duration: int = 300,
470
frame_recovery: bool = False,
471
+ *,
472
+ backend: Literal["opencv", "pyav"] = "opencv",
473
**kwargs,
474
) -> tuple[npt.NDArray, dict[str, Any]]:
400
- """
401
- Load video frames from bytes.
475
+ """Load sampled frames from raw video bytes.
476
477
Args:
404
- data: Raw video bytes
405
- num_frames: Target number of frames to sample (-1 for all)
406
- fps: Target FPS for sampling (-1 for original)
407
- max_duration: Maximum duration (unused in base backend)
408
- frame_recovery: Enable forward-scan recovery for failed frames
478
+ data: Raw video bytes.
479
+ num_frames: Target number of frames to sample (``-1`` for all).
480
+ fps: Target FPS for sampling (``-1`` for original).
481
+ max_duration: Maximum duration in seconds — only used by the
482
+ dynamic subclass; ignored here.
483
+ frame_recovery: Enable forward-scan recovery for failed frames.
484
+ Only honored by the OpenCV codec.
485
+ backend: Decoding codec — ``"opencv"`` or ``"pyav"`` .
486
487
Returns:
411
- Tuple of (frames_array, metadata_dict)
488
+ Tuple of ``(frames_array, metadata_dict)``.
489
"""
413
- cap = cls.open_video_capture(data)
414
-
415
- source = OpenCVVideoBackendMixin.get_video_metadata(cap)
490
target = VideoTargetMetadata(
417
- num_frames=num_frames,
418
- fps=fps,
419
- max_duration=max_duration,
491
+ num_frames=num_frames, fps=fps, max_duration=max_duration
492
)
493
422
- # resample video to target num_frames and fps
423
- # - the minimum of the two will be used
424
- frame_idx = cls.compute_frames_index_to_sample(
425
- source=source,
426
- target=target,
427
- )
494
+ if backend == "opencv":
495
+ cap = cls.open_video_capture(data)
496
+ source = cls._prepare_source(cls.get_video_metadata(cap))
497
+ frame_idx = cls.compute_frames_index_to_sample(
498
+ source=source, target=target, **kwargs
499
+ )
500
+ frames, valid = cls.read_frames(
501
+ cap,
502
+ frame_idx,
503
+ total_frames_num=source.total_frames_num,
504
+ frame_recovery=frame_recovery,
505
+ )
506
+ elif backend == "pyav":
507
+ assert not frame_recovery, (
508
+ "frame_recovery is only available for `opencv` backend"
509
+ )
510
+ with av.open(BytesIO(data)) as container:
511
+ source = cls._prepare_source(cls.get_metadata(container))
512
+ frame_idx = cls.compute_frames_index_to_sample(
513
+ source=source, target=target, **kwargs
514
+ )
515
+ frames, valid = cls.decode_frames(
516
+ container, frame_idx, source.original_fps, source.duration
517
+ )
518
+ else:
519
+ raise ValueError(
520
+ f"Unknown video codec backend {backend!r}; "
521
+ "valid options: 'opencv', 'pyav'."
522
+ )
523
429
- frames, valid_frame_indices = cls.read_frames(
430
- cap,
431
- frame_idx,
432
- total_frames_num=source.total_frames_num,
433
- frame_recovery=frame_recovery,
434
- )
524
+ if len(valid) < len(frame_idx):
525
+ logger.warning(
526
+ "%s video loading: expected %d frames but got %d.",
527
+ backend,
528
+ len(frame_idx),
529
+ len(valid),
530
+ )
531
436
- metadata = cls.create_hf_metadata(
532
+ return frames, cls.create_hf_metadata(
533
source=source,
438
- video_backend="opencv",
439
- valid_frame_indices=valid_frame_indices,
534
+ video_backend=f"{backend}{cls._sampling_suffix}",
535
+ valid_frame_indices=valid,
536
)
537
442
- return frames, metadata
443
-
538
539
@VIDEO_LOADER_REGISTRY.register("opencv_dynamic")
446
-class OpenCVDynamicVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
540
+class DynamicVideoBackend(VideoBackend):
541
+ """Duration-aware dynamic-sampling video backend.
542
+
543
+ Samples at ``fps`` up to ``max_duration`` seconds, falling back to
544
+ uniform sampling across the full duration when the video is longer
545
+ than ``max_duration``. Codec is selectable the same way as
546
+ :class:`VideoBackend`.
547
+ """
548
+
549
+ _sampling_suffix: ClassVar[str] = "_dynamic"
550
+
551
+ @classmethod
552
+ def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:
553
+ # Estimate duration from frame count and fps when the container
554
+ # does not report it (common for WebM/streaming inputs).
555
+ if source.duration:
556
+ return source
557
+ if source.original_fps > 0:
558
+ max_frame_idx = source.total_frames_num - 1
559
+ duration = round(max_frame_idx / source.original_fps) + 1
560
+ else:
561
+ duration = 0
562
+ return VideoSourceMetadata(
563
+ source.total_frames_num, source.original_fps, duration
564
+ )
565
+
566
@classmethod
567
def compute_frames_index_to_sample(
568
cls,
@@ -456,8 +575,8 @@ class OpenCVDynamicVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
575
original_fps = source.original_fps
576
max_duration = target.max_duration
577
fps = target.fps
459
-
578
max_frame_idx = source.total_frames_num - 1
579
+
580
# Refer to:
581
# https://github.com/huggingface/transformers/blob/v4.55.4/src/transformers/models/glm4v/video_processing_glm4v.py#L103-L140
582
frame_indices_list: list[int]
@@ -491,62 +610,20 @@ class OpenCVDynamicVideoBackend(VideoLoader, OpenCVVideoBackendMixin):
610
fps: int = 2,
611
max_duration: int = 300,
612
frame_recovery: bool = False,
613
+ *,
614
+ backend: Literal["opencv", "pyav"] = "opencv",
615
**kwargs,
616
) -> tuple[npt.NDArray, dict[str, Any]]:
496
- """
497
- Load video frames with dynamic sampling based on duration.
498
-
499
- Args:
500
- data: Raw video bytes
501
- num_frames: Not used in dynamic backend
502
- fps: Target FPS for sampling (default: 2)
503
- max_duration: Maximum video duration to process (default: 300s)
504
- frame_recovery: Enable forward-scan recovery for failed frames
505
-
506
- Returns:
507
- Tuple of (frames_array, metadata_dict)
508
- """
509
- cap = cls.open_video_capture(data)
510
-
511
- orig_source = OpenCVVideoBackendMixin.get_video_metadata(cap)
512
- max_frame_idx = orig_source.total_frames_num - 1
513
- duration = (
514
- orig_source.duration or round(max_frame_idx / orig_source.original_fps) + 1
515
- )
516
-
517
- # recompute source metadata with adjusted duration to ensure correct
518
- # sampling indices computation
519
- source = VideoSourceMetadata(
520
- total_frames_num=orig_source.total_frames_num,
521
- original_fps=orig_source.original_fps,
522
- duration=duration,
523
- )
524
- target = VideoTargetMetadata(
617
+ return super().load_bytes(
618
+ data,
619
num_frames=num_frames,
620
fps=fps,
621
max_duration=max_duration,
528
- )
529
-
530
- frame_indices_list = cls.compute_frames_index_to_sample(
531
- source=source,
532
- target=target,
533
- )
534
-
535
- frames, valid_frame_indices = cls.read_frames(
536
- cap,
537
- frame_indices_list,
538
- total_frames_num=source.total_frames_num,
622
frame_recovery=frame_recovery,
623
+ backend=backend,
624
+ **kwargs,
625
)
626
542
- metadata = cls.create_hf_metadata(
543
- source=source,
544
- video_backend="opencv_dynamic",
545
- valid_frame_indices=valid_frame_indices,
546
- )
547
-
548
- return frames, metadata
549
-
627
628
@VIDEO_LOADER_REGISTRY.register("molmo2")
629
class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin):
@@ -835,7 +912,7 @@ class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin):
912
913
914
@VIDEO_LOADER_REGISTRY.register("nemotron_vl")
838
-class NemotronVLVideoBackend(OpenCVVideoBackend):
915
+class NemotronVLVideoBackend(VideoBackend):
916
@classmethod
917
def load_bytes(
918
cls,
@@ -844,14 +921,17 @@ class NemotronVLVideoBackend(OpenCVVideoBackend):
921
fps: int = -1,
922
max_duration: int = 300,
923
frame_recovery: bool = False,
924
+ *,
925
+ backend: Literal["opencv", "pyav"] = "opencv",
926
**kwargs,
927
) -> tuple[npt.NDArray, dict[str, Any]]:
849
- frames, metadata = OpenCVVideoBackend.load_bytes(
928
+ frames, metadata = super().load_bytes(
929
data,
930
num_frames=num_frames,
931
fps=fps,
932
max_duration=max_duration,
933
frame_recovery=frame_recovery,
934
+ backend=backend,
935
**kwargs,
936
)
937