@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e4ae148a78

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 02:45 UTC

[Refactor] Modular video loader backend refactoring (#35202)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>

Isotr0py committed Mar 6, 2026 at 22:06 UTC e4ae148a787df846beb194078c35655c44784bd5

2 files changed +444 -267

tests/multimodal/test_video.py

+80 -1

@@ -7,7 +7,13 @@ import numpy as np

7 import numpy.typing as npt

8 import pytest

10 -from vllm.multimodal.video import VIDEO_LOADER_REGISTRY, VideoLoader

10 +from vllm.assets.base import get_vllm_public_assets

11 +from vllm.multimodal.video import (

12 + VIDEO_LOADER_REGISTRY,

13 + VideoLoader,

14 +)

15 +

16 +from .utils import create_video_from_image

18 pytestmark = pytest.mark.cpu_test

@@ -291,3 +297,76 @@ def test_video_recovery_dynamic_backend(monkeypatch: pytest.MonkeyPatch):

297 f"Got {frames_with_recovery.shape[0]} with recovery vs "

298 f"{frames_no_recovery.shape[0]} without"

299 )

300 +

301 +

302 +@pytest.fixture

303 +def dummy_video_path(tmp_path):

304 + image_path = get_vllm_public_assets(

305 + filename="stop_sign.jpg", s3_prefix="vision_model_images"

306 + )

307 +

308 + video_path = tmp_path / "test_RGB_video.mp4"

309 + create_video_from_image(str(image_path), str(video_path), num_frames=1800, fps=30)

310 + return video_path

311 +

312 +

313 +@pytest.mark.parametrize(

314 + "backend, kwargs, expected_num_frames",

315 + [

316 + # opencv: num_frames directly controls count

317 + pytest.param("opencv", {"num_frames": 32}, 32, id="opencv-num_frames"),

318 + pytest.param("opencv", {"fps": 2}, 120, id="opencv-fps"),

319 + pytest.param(

320 + "opencv",

321 + {"num_frames": 500, "fps": 2},

322 + 120,

323 + id="opencv-num_frames_wins_fps",

324 + ),

325 + pytest.param(

326 + "opencv_dynamic",

327 + {"fps": 1, "max_duration": 60},

328 + 60,

329 + id="opencv_dynamic-within_max_duration",

330 + ),

331 + pytest.param(

332 + "opencv_dynamic",

333 + {"fps": 2, "max_duration": 30},

334 + 60,

335 + id="opencv_dynamic-exceeds_max_duration",

336 + ),

337 + pytest.param(

338 + "openpangu", {"num_frames": 32, "fps": -1}, 32, id="openpangu-num_frames"

339 + ),

340 + pytest.param(

341 + "molmo2",

342 + {"num_frames": 32, "frame_sample_mode": "uniform_last_frame"},

343 + 32,

344 + id="molmo2-uniform_last_frame",

345 + ),

346 + pytest.param(

347 + "molmo2",

348 + {"fps": 2, "frame_sample_mode": "fps"},

349 + 119,

350 + id="molmo2-fps",

351 + ),

352 + ],

353 +)

354 +def test_video_loader_frames_sampling(

355 + dummy_video_path,

356 + monkeypatch: pytest.MonkeyPatch,

357 + backend: str,

358 + kwargs: dict,

359 + expected_num_frames: int,

360 +):

361 + """Test video loader frames sampling functionality."""

362 + monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", backend)

363 + loader = VIDEO_LOADER_REGISTRY.load(backend)

364 +

365 + with open(dummy_video_path, "rb") as f:

366 + long_video_bytes = f.read()

367 +

368 + frames, _ = loader.load_bytes(long_video_bytes, **kwargs)

369 +

370 + assert frames.ndim == 4

371 + assert frames.shape[3] == 3 # RGB

372 + assert frames.shape[0] == expected_num_frames

vllm/multimodal/video.py

+364 -266

@@ -3,17 +3,23 @@

3 import math

4 from abc import abstractmethod

5 from io import BytesIO

6 -from typing import TYPE_CHECKING, Any, cast

6 +from typing import Any, NamedTuple, cast

8 import numpy as np

9 import numpy.typing as npt

11 -if TYPE_CHECKING:

12 - import cv2

13 -

11 from vllm.logger import init_logger

12 +from vllm.utils.import_utils import PlaceholderModule

13 from vllm.utils.registry import ExtensionManager

15 +try:

16 + import cv2

17 + import cv2.videoio_registry as vr

18 +except ImportError:

19 + cv2 = PlaceholderModule("cv2")

20 + vr = PlaceholderModule("cv2").placeholder_attr("videoio_registry")

21 +

22 +

23 logger = init_logger(__name__)

@@ -23,8 +29,6 @@ def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray:

29 resized_frames = np.empty(

30 (num_frames, new_height, new_width, channels), dtype=frames.dtype

31 )

26 - # lazy import cv2 to avoid bothering users who only use text models

27 - import cv2

33 for i, frame in enumerate(frames):

34 resized_frame = cv2.resize(frame, (new_width, new_height))

@@ -50,16 +54,100 @@ def sample_frames_from_video(frames: npt.NDArray, num_frames: int) -> npt.NDArra

54 return sampled_frames

57 +class VideoTargetMetadata(NamedTuple):

58 + """Metadata represents target video."""

59 +

60 + num_frames: int

61 + fps: float

62 + max_duration: float

63 +

64 +

65 +class VideoSourceMetadata(NamedTuple):

66 + """Metadata represents source video."""

67 +

68 + total_frames_num: int

69 + original_fps: float

70 + duration: float

71 +

72 +

73 class VideoLoader:

74 + @classmethod

75 + def compute_frames_index_to_sample(

76 + cls,

77 + source: VideoSourceMetadata,

78 + target: VideoTargetMetadata,

79 + **kwargs,

80 + ) -> list[int]:

81 + """Return the list of frame indices to sample from the video."""

82 + raise NotImplementedError

83 +

84 @classmethod

85 @abstractmethod

86 def load_bytes(

57 - cls, data: bytes, num_frames: int = -1, **kwargs

87 + cls,

88 + data: bytes,

89 + **kwargs,

90 ) -> tuple[npt.NDArray, dict[str, Any]]:

91 + """Load video frames from bytes and return (frames_array, metadata_dict)."""

92 raise NotImplementedError

94 + @classmethod

95 + def create_hf_metadata(

96 + cls,

97 + source: VideoSourceMetadata,

98 + valid_frame_indices: list[int],

99 + video_backend: str,

100 + ):

101 + return {

102 + "total_num_frames": source.total_frames_num,

103 + "fps": source.original_fps,

104 + "duration": source.duration,

105 + "video_backend": video_backend,

106 + "frames_indices": valid_frame_indices,

107 + "do_sample_frames": len(valid_frame_indices) == source.total_frames_num,

108 + }

109 +

110 +

111 +VIDEO_LOADER_REGISTRY = ExtensionManager()

112 +

113 +

114 +class OpenCVVideoBackendMixin:

115 @staticmethod

116 + def get_cv2_video_api():

117 + api_pref = None

118 + for backend in vr.getStreamBufferedBackends():

119 + if not vr.hasBackend(backend):

120 + continue

121 + if not vr.isBackendBuiltIn(backend):

122 + _, abi, api = vr.getStreamBufferedBackendPluginVersion(backend)

123 + if abi < 1 or (abi == 1 and api < 2):

124 + continue

125 + api_pref = backend

126 + break

127 + return api_pref

128 +

129 + @classmethod

130 + def open_video_capture(cls, data: bytes) -> "cv2.VideoCapture":

131 + backend = cls.get_cv2_video_api()

132 + cap = cv2.VideoCapture(BytesIO(data), backend, [])

133 + if not cap.isOpened():

134 + raise ValueError("Could not open video stream")

135 + return cap

136 +

137 + @staticmethod

138 + def get_video_metadata(cap: "cv2.VideoCapture") -> VideoSourceMetadata:

139 + total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

140 + original_fps = cap.get(cv2.CAP_PROP_FPS)

141 + duration = total_frames_num / original_fps if original_fps > 0 else 0

142 + return VideoSourceMetadata(

143 + total_frames_num=total_frames_num,

144 + original_fps=original_fps,

145 + duration=duration,

146 + )

147 +

148 + @classmethod

149 def _can_use_for_recovery(

150 + cls,

151 idx: int,

152 failed_frames: list[int],

153 next_target_map: dict[int, int],

@@ -72,8 +160,9 @@ class VideoLoader:

160 limit = next_target_map.get(oldest_failed, total_frames)

161 return idx < limit

162

75 - @staticmethod

163 + @classmethod

164 def _read_frames_with_recovery(

165 + cls,

166 cap: "cv2.VideoCapture",

167 frame_indices: list[int],

168 total_frames: int,

@@ -95,8 +184,6 @@ class VideoLoader:

184 - valid_frame_indices: List of frame indices that were loaded

185 - recovered_map: Dict mapping recovered_idx -> source_idx

186 """

98 - import cv2

99 -

187 width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))

188 height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

189

@@ -135,7 +222,7 @@ class VideoLoader:

222 continue

223

224 # Check if we should retrieve: target frame OR can recover a failed one

138 - can_recover = VideoLoader._can_use_for_recovery(

225 + can_recover = cls._can_use_for_recovery(

226 idx, failed_frames_idx, next_target_map, total_frames

227 )

228

@@ -179,15 +266,14 @@ class VideoLoader:

266

267 return frames, valid_frame_indices, recovered_map

268

182 - @staticmethod

183 - def _read_frames(

269 + @classmethod

270 + def _read_frames_no_recovery(

271 + cls,

272 cap,

273 frame_indices: set[int],

186 - num_expected_frames: int,

274 max_frame_idx: int,

188 - ) -> tuple[npt.NDArray, int, list[int]]:

189 - import cv2

190 -

275 + ) -> tuple[npt.NDArray, list[int]]:

276 + num_expected_frames = len(frame_indices)

277 width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))

278 height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))

279 frames = np.empty((num_expected_frames, height, width, 3), dtype=np.uint8)

@@ -229,63 +315,60 @@ class VideoLoader:

315 valid_num_frames,

316 )

317

232 - return frames[:valid_num_frames], valid_num_frames, valid_frame_indices

318 + return frames[:valid_num_frames], valid_frame_indices

319

320 + @classmethod

321 + def read_frames(

322 + cls,

323 + cap: "cv2.VideoCapture",

324 + frame_idx: list[int],

325 + total_frames_num: int,

326 + *,

327 + frame_recovery: bool = False,

328 + ) -> tuple[npt.NDArray, list[int]]:

329 + if frame_recovery:

330 + num_frames_to_sample = len(frame_idx)

331 + frames, valid_frame_indices, recovered_map = cls._read_frames_with_recovery(

332 + cap, frame_idx, total_frames_num

333 + )

334

235 -VIDEO_LOADER_REGISTRY = ExtensionManager()

335 + if recovered_map:

336 + logger.info(

337 + "Frame recovery: %d frames recovered using forward scan.",

338 + len(recovered_map),

339 + )

340 + else:

341 + frame_idx_set = set(frame_idx)

342 + num_frames_to_sample = len(frame_idx_set)

343 + frames, valid_frame_indices = cls._read_frames_no_recovery(

344 + cap, frame_idx_set, max(frame_idx)

345 + )

346 + valid_num_frames = len(valid_frame_indices)

347 + if valid_num_frames < num_frames_to_sample:

348 + logger.warning(

349 + "Video loading completed with %d broken/unreadable frames. "

350 + "Expected to sample %d frames but only loaded %d frames.",

351 + num_frames_to_sample - valid_num_frames,

352 + num_frames_to_sample,

353 + valid_num_frames,

354 + )

355 + return frames, valid_frame_indices

356

357

358 @VIDEO_LOADER_REGISTRY.register("opencv")

239 -class OpenCVVideoBackend(VideoLoader):

240 - def get_cv2_video_api(self):

241 - import cv2.videoio_registry as vr

242 -

243 - api_pref = None

244 - for backend in vr.getStreamBufferedBackends():

245 - if not vr.hasBackend(backend):

246 - continue

247 - if not vr.isBackendBuiltIn(backend):

248 - _, abi, api = vr.getStreamBufferedBackendPluginVersion(backend)

249 - if abi < 1 or (abi == 1 and api < 2):

250 - continue

251 - api_pref = backend

252 - break

253 - return api_pref

254 -

359 +class OpenCVVideoBackend(VideoLoader, OpenCVVideoBackendMixin):

360 @classmethod

256 - def load_bytes(

361 + def compute_frames_index_to_sample(

362 cls,

258 - data: bytes,

259 - num_frames: int = -1,

260 - fps: int = -1,

261 - max_duration: int = 300,

262 - frame_recovery: bool = False,

363 + source: VideoSourceMetadata,

364 + target: VideoTargetMetadata,

365 **kwargs,

264 - ) -> tuple[npt.NDArray, dict[str, Any]]:

265 - """

266 - Load video frames from bytes.

267 -

268 - Args:

269 - data: Raw video bytes

270 - num_frames: Target number of frames to sample (-1 for all)

271 - fps: Target FPS for sampling (-1 for original)

272 - max_duration: Maximum duration (unused in base backend)

273 - frame_recovery: Enable forward-scan recovery for failed frames

274 -

275 - Returns:

276 - Tuple of (frames_array, metadata_dict)

277 - """

278 - import cv2

279 -

280 - backend = cls().get_cv2_video_api()

281 - cap = cv2.VideoCapture(BytesIO(data), backend, [])

282 - if not cap.isOpened():

283 - raise ValueError("Could not open video stream")

284 -

285 - total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

286 - original_fps = cap.get(cv2.CAP_PROP_FPS)

287 - duration = total_frames_num / original_fps if original_fps > 0 else 0

366 + ) -> list[int]:

367 + total_frames_num = source.total_frames_num

368 + duration = source.duration

369

370 + num_frames = target.num_frames

371 + fps = target.fps

372 # resample video to target num_frames and fps

373 # - the minimum of the two will be used

374 num_frames_to_sample = total_frames_num

@@ -302,81 +385,79 @@ class OpenCVVideoBackend(VideoLoader):

385 0, total_frames_num - 1, num_frames_to_sample, dtype=int

386 )

387 frame_idx = uniform_sampled_frames.tolist()

388 + return frame_idx

389

306 - if frame_recovery:

307 - frames, valid_frame_indices, recovered_map = cls._read_frames_with_recovery(

308 - cap, frame_idx, total_frames_num

309 - )

310 - valid_num_frames = len(valid_frame_indices)

311 -

312 - if recovered_map:

313 - logger.info(

314 - "Frame recovery: %d frames recovered using forward scan.",

315 - len(recovered_map),

316 - )

317 - else:

318 - frame_idx_set = set(frame_idx)

319 - frames, valid_num_frames, valid_frame_indices = cls._read_frames(

320 - cap, frame_idx_set, num_frames_to_sample, max(frame_idx)

321 - )

322 -

323 - # Use transformers transformers.video_utils.VideoMetadata format

324 - # NOTE(Isotr0py): For models like Qwen3-VL/GLM4.5V, this metadata

325 - # can cause incorrect timestamp calculation without num_frames=-1.

326 - metadata = {

327 - "total_num_frames": total_frames_num,

328 - "fps": original_fps,

329 - "duration": duration,

330 - "video_backend": "opencv",

331 - "frames_indices": valid_frame_indices,

332 - # extra field used to control hf processor's video

333 - # sampling behavior

334 - "do_sample_frames": valid_num_frames == total_frames_num,

335 - }

336 -

337 - return frames, metadata

338 -

339 -

340 -@VIDEO_LOADER_REGISTRY.register("opencv_dynamic")

341 -class OpenCVDynamicVideoBackend(OpenCVVideoBackend):

390 @classmethod

391 def load_bytes(

392 cls,

393 data: bytes,

394 num_frames: int = -1,

347 - fps: int = 2,

395 + fps: int = -1,

396 max_duration: int = 300,

397 frame_recovery: bool = False,

398 **kwargs,

399 ) -> tuple[npt.NDArray, dict[str, Any]]:

400 """

353 - Load video frames with dynamic sampling based on duration.

401 + Load video frames from bytes.

402

403 Args:

404 data: Raw video bytes

357 - num_frames: Not used in dynamic backend

358 - fps: Target FPS for sampling (default: 2)

359 - max_duration: Maximum video duration to process (default: 300s)

405 + num_frames: Target number of frames to sample (-1 for all)

406 + fps: Target FPS for sampling (-1 for original)

407 + max_duration: Maximum duration (unused in base backend)

408 frame_recovery: Enable forward-scan recovery for failed frames

409

410 Returns:

411 Tuple of (frames_array, metadata_dict)

412 """

365 - import cv2

413 + cap = cls.open_video_capture(data)

414

367 - backend = cls().get_cv2_video_api()

368 - cap = cv2.VideoCapture(BytesIO(data), backend, [])

369 - if not cap.isOpened():

370 - raise ValueError("Could not open video stream")

415 + source = OpenCVVideoBackendMixin.get_video_metadata(cap)

416 + target = VideoTargetMetadata(

417 + num_frames=num_frames,

418 + fps=fps,

419 + max_duration=max_duration,

420 + )

421

372 - total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

373 - original_fps = cap.get(cv2.CAP_PROP_FPS)

374 - duration = total_frames_num / original_fps if original_fps > 0 else 0

422 + # resample video to target num_frames and fps

423 + # - the minimum of the two will be used

424 + frame_idx = cls.compute_frames_index_to_sample(

425 + source=source,

426 + target=target,

427 + )

428 +

429 + frames, valid_frame_indices = cls.read_frames(

430 + cap,

431 + frame_idx,

432 + total_frames_num=source.total_frames_num,

433 + frame_recovery=frame_recovery,

434 + )

435

376 - # resample video to target num_frames

377 - max_frame_idx = total_frames_num - 1

378 - duration = duration or round(max_frame_idx / original_fps) + 1

436 + metadata = cls.create_hf_metadata(

437 + source=source,

438 + video_backend="opencv",

439 + valid_frame_indices=valid_frame_indices,

440 + )

441

442 + return frames, metadata

443 +

444 +

445 +@VIDEO_LOADER_REGISTRY.register("opencv_dynamic")

446 +class OpenCVDynamicVideoBackend(VideoLoader, OpenCVVideoBackendMixin):

447 + @classmethod

448 + def compute_frames_index_to_sample(

449 + cls,

450 + source: VideoSourceMetadata,

451 + target: VideoTargetMetadata,

452 + **kwargs,

453 + ) -> list[int]:

454 + total_frames_num = source.total_frames_num

455 + duration = source.duration

456 + original_fps = source.original_fps

457 + max_duration = target.max_duration

458 + fps = target.fps

459 +

460 + max_frame_idx = source.total_frames_num - 1

461 # Refer to:

462 # https://github.com/huggingface/transformers/blob/v4.55.4/src/transformers/models/glm4v/video_processing_glm4v.py#L103-L140

463 frame_indices_list: list[int]

@@ -400,54 +481,75 @@ class OpenCVDynamicVideoBackend(OpenCVVideoBackend):

481 for t in target_seconds

482 }

483 )

484 + return frame_indices_list

485

404 - if frame_recovery:

405 - frames, valid_frame_indices, recovered_map = cls._read_frames_with_recovery(

406 - cap, frame_indices_list, total_frames_num

407 - )

408 - valid_num_frames = len(valid_frame_indices)

486 + @classmethod

487 + def load_bytes(

488 + cls,

489 + data: bytes,

490 + num_frames: int = -1,

491 + fps: int = 2,

492 + max_duration: int = 300,

493 + frame_recovery: bool = False,

494 + **kwargs,

495 + ) -> tuple[npt.NDArray, dict[str, Any]]:

496 + """

497 + Load video frames with dynamic sampling based on duration.

498

410 - if recovered_map:

411 - logger.info(

412 - "Frame recovery: %d frames recovered using forward scan.",

413 - len(recovered_map),

414 - )

415 - else:

416 - frame_indices_set = set(frame_indices_list)

417 - frames, valid_num_frames, valid_frame_indices = cls._read_frames(

418 - cap, frame_indices_set, len(frame_indices_list), total_frames_num - 1

419 - )

499 + Args:

500 + data: Raw video bytes

501 + num_frames: Not used in dynamic backend

502 + fps: Target FPS for sampling (default: 2)

503 + max_duration: Maximum video duration to process (default: 300s)

504 + frame_recovery: Enable forward-scan recovery for failed frames

505

421 - # Use transformers transformers.video_utils.VideoMetadata format

422 - metadata = {

423 - "total_num_frames": total_frames_num,

424 - "fps": original_fps,

425 - "duration": duration,

426 - "video_backend": "opencv_dynamic",

427 - "frames_indices": valid_frame_indices,

428 - "do_sample_frames": False,

429 - }

506 + Returns:

507 + Tuple of (frames_array, metadata_dict)

508 + """

509 + cap = cls.open_video_capture(data)

510

431 - return frames, metadata

511 + orig_source = OpenCVVideoBackendMixin.get_video_metadata(cap)

512 + max_frame_idx = orig_source.total_frames_num - 1

513 + duration = (

514 + orig_source.duration or round(max_frame_idx / orig_source.original_fps) + 1

515 + )

516

517 + # recompute source metadata with adjusted duration to ensure correct

518 + # sampling indices computation

519 + source = VideoSourceMetadata(

520 + total_frames_num=orig_source.total_frames_num,

521 + original_fps=orig_source.original_fps,

522 + duration=duration,

523 + )

524 + target = VideoTargetMetadata(

525 + num_frames=num_frames,

526 + fps=fps,

527 + max_duration=max_duration,

528 + )

529

434 -@VIDEO_LOADER_REGISTRY.register("molmo2")

435 -class Molmo2VideoBackend(VideoLoader):

436 - def get_cv2_video_api(self):

437 - import cv2.videoio_registry as vr

530 + frame_indices_list = cls.compute_frames_index_to_sample(

531 + source=source,

532 + target=target,

533 + )

534 +

535 + frames, valid_frame_indices = cls.read_frames(

536 + cap,

537 + frame_indices_list,

538 + total_frames_num=source.total_frames_num,

539 + frame_recovery=frame_recovery,

540 + )

541 +

542 + metadata = cls.create_hf_metadata(

543 + source=source,

544 + video_backend="opencv_dynamic",

545 + valid_frame_indices=valid_frame_indices,

546 + )

547 +

548 + return frames, metadata

549

439 - api_pref = None

440 - for backend in vr.getStreamBufferedBackends():

441 - if not vr.hasBackend(backend):

442 - continue

443 - if not vr.isBackendBuiltIn(backend):

444 - _, abi, api = vr.getStreamBufferedBackendPluginVersion(backend)

445 - if abi < 1 or (abi == 1 and api < 2):

446 - continue

447 - api_pref = backend

448 - break

449 - return api_pref

550

551 +@VIDEO_LOADER_REGISTRY.register("molmo2")

552 +class Molmo2VideoBackend(VideoLoader, OpenCVVideoBackendMixin):

553 @classmethod

554 def get_candidate_target_fps(

555 cls,

@@ -599,16 +701,28 @@ class Molmo2VideoBackend(VideoLoader):

701 raise NotImplementedError(frame_sample_mode)

702

703 @classmethod

602 - def _sample_frames(

704 + def compute_frames_index_to_sample(

705 cls,

604 - total_num_frames: int,

605 - video_fps: float,

606 - duration: float,

607 - frame_sample_mode: str,

608 - num_frames: int,

609 - max_fps: int,

610 - sampling_fps: int,

611 - ) -> npt.NDArray:

706 + source: VideoSourceMetadata,

707 + target: VideoTargetMetadata,

708 + **kwargs,

709 + ):

710 + max_fps = kwargs.get("max_fps")

711 + frame_sample_mode = kwargs.get("frame_sample_mode")

712 + if frame_sample_mode is None:

713 + return list(range(0, source.total_frames_num))

714 +

715 + if frame_sample_mode not in {"uniform_last_frame", "fps"}:

716 + raise NotImplementedError(

717 + f"Unsupported frame_sample_mode: {frame_sample_mode}"

718 + )

719 +

720 + duration = source.duration

721 + video_fps = source.original_fps

722 + total_num_frames = source.total_frames_num

723 + num_frames = target.num_frames

724 + sampling_fps = target.fps

725 +

726 if frame_sample_mode == "uniform_last_frame" and max_fps is not None:

727 if total_num_frames <= 2:

728 indices = np.arange(total_num_frames).astype(int)

@@ -655,10 +769,7 @@ class Molmo2VideoBackend(VideoLoader):

769 num_frames,

770 video_fps,

771 )

658 - else:

659 - raise NotImplementedError(frame_sample_mode)

660 -

661 - return indices

772 + return indices.tolist()

773

774 @classmethod

775 def load_bytes_opencv(

@@ -668,63 +779,37 @@ class Molmo2VideoBackend(VideoLoader):

779 num_frames: int = -1,

780 max_fps: int = 2,

781 sampling_fps: int = 2,

782 + frame_recovery: bool = False,

783 **kwargs,

784 ) -> tuple[npt.NDArray, dict[str, Any]]:

673 - import cv2

785 + cap = cls.open_video_capture(data)

786

675 - backend = cls().get_cv2_video_api()

676 - cap = cv2.VideoCapture(BytesIO(data), backend, [])

677 - if not cap.isOpened():

678 - raise ValueError("Could not open video stream")

679 -

680 - total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

681 - original_fps = cap.get(cv2.CAP_PROP_FPS)

682 - duration = total_frames_num / original_fps if original_fps > 0 else 0

787 + source = OpenCVVideoBackendMixin.get_video_metadata(cap)

788 + target = VideoTargetMetadata(

789 + num_frames=num_frames,

790 + fps=sampling_fps,

791 + max_duration=source.duration,

792 + )

793

684 - if frame_sample_mode is None:

685 - # Use transformers transformers.video_utils.VideoMetadata format

686 - frame_idx = list(range(0, total_frames_num))

687 - frame_idx_set = set(frame_idx)

688 - frames, valid_num_frames, valid_frame_indices = cls._read_frames(

689 - cap, frame_idx_set, total_frames_num, max(frame_idx)

690 - )

691 - do_sample_frames = valid_num_frames == total_frames_num

692 - metadata = {

693 - "total_num_frames": total_frames_num,

694 - "fps": original_fps,

695 - "duration": duration,

696 - "video_backend": "opencv",

697 - "do_sample_frames": do_sample_frames,

698 - }

699 - if not do_sample_frames:

700 - metadata["frames_indices"] = valid_frame_indices

701 - return frames, metadata

702 -

703 - frame_idx = cls._sample_frames(

704 - total_frames_num,

705 - original_fps,

706 - duration,

707 - frame_sample_mode,

708 - num_frames,

709 - max_fps,

710 - sampling_fps,

711 - ).tolist()

794 + frame_idx = cls.compute_frames_index_to_sample(

795 + source=source,

796 + target=target,

797 + frame_sample_mode=frame_sample_mode,

798 + max_fps=max_fps,

799 + )

800

713 - frames, valid_num_frames, valid_frame_indices = cls._read_frames(

801 + frames, valid_frame_indices = cls.read_frames(

802 cap,

715 - set(frame_idx),

716 - len(frame_idx),

717 - total_frames_num - 1,

803 + frame_idx,

804 + total_frames_num=source.total_frames_num,

805 + frame_recovery=frame_recovery,

806 )

807

720 - metadata = {

721 - "total_num_frames": total_frames_num,

722 - "fps": original_fps,

723 - "duration": duration,

724 - "video_backend": "opencv",

725 - "frames_indices": valid_frame_indices,

726 - "do_sample_frames": False,

727 - }

808 + metadata = cls.create_hf_metadata(

809 + source=source,

810 + video_backend="opencv",

811 + valid_frame_indices=valid_frame_indices,

812 + )

813

814 return frames, metadata

815

@@ -777,42 +862,19 @@ class NemotronVLVideoBackend(OpenCVVideoBackend):

862

863

864 @VIDEO_LOADER_REGISTRY.register("openpangu")

780 -class OpenCVDynamicOpenPanguVideoBackend(OpenCVVideoBackend):

865 +class OpenCVDynamicOpenPanguVideoBackend(VideoLoader, OpenCVVideoBackendMixin):

866 @classmethod

782 - def load_bytes(

867 + def compute_frames_index_to_sample(

868 cls,

784 - data: bytes,

785 - num_frames: int = 32,

786 - fps: int = 1,

787 - max_duration: int = 300,

788 - frame_recovery: bool = False,

869 + source: VideoSourceMetadata,

870 + target: VideoTargetMetadata,

871 **kwargs,

790 - ) -> tuple[npt.NDArray, dict[str, Any]]:

791 - """

792 - Load video frames with dynamic sampling based on duration.

793 - Assume that total_num_frames = 10 and fps = 1.

794 - The timestamp of frame 0 is 0.0.

795 - The timestamp of frame 1 is 1.0.…

796 - The timestamp of frame 9 (the last frame) should be 9.0, that is,

797 - (total_frames_num – 1) / original_fps.

798 -

799 - Args:

800 - data: Raw video bytes

801 - num_frames: Not used in dynamic backend

802 - fps: Target FPS for sampling (default: 1)

803 -

804 - Returns:

805 - Tuple of (frames_array, metadata_dict)

806 - """

807 - import cv2

808 -

809 - backend = cls().get_cv2_video_api()

810 - cap = cv2.VideoCapture(BytesIO(data), backend, [])

811 - if not cap.isOpened():

812 - raise ValueError("Could not open video stream")

872 + ) -> list[int]:

873 + total_frames_num = source.total_frames_num

874 + original_fps = source.original_fps

875 + num_frames = target.num_frames

876 + fps = target.fps

877

814 - total_frames_num = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

815 - original_fps = float(cap.get(cv2.CAP_PROP_FPS))

878 # The timestamp of the rightmost frame, cannot be used to calculate frame 0.

879 if total_frames_num >= 1 and original_fps > 0:

880 total_duration = (total_frames_num - 1) / original_fps

@@ -841,23 +903,59 @@ class OpenCVDynamicOpenPanguVideoBackend(OpenCVVideoBackend):

903 min(total_frames_num - 1, round(t * original_fps))

904 for t in sample_frame_timestamps

905 ]

906 + return frames_indices

907 +

908 + @classmethod

909 + def load_bytes(

910 + cls,

911 + data: bytes,

912 + num_frames: int = -1,

913 + fps: int = 2,

914 + max_duration: int = 300,

915 + frame_recovery: bool = False,

916 + **kwargs,

917 + ) -> tuple[npt.NDArray, dict[str, Any]]:

918 + """

919 + Load video frames with dynamic sampling based on duration.

920 +

921 + Args:

922 + data: Raw video bytes

923 + num_frames: Not used in dynamic backend

924 + fps: Target FPS for sampling (default: 2)

925 + max_duration: Maximum video duration to process (default: 300s)

926 + frame_recovery: Enable forward-scan recovery for failed frames

927

845 - frames, valid_frame_indices, recovered_map = cls._read_frames_with_recovery(

846 - cap, frames_indices, total_frames_num

928 + Returns:

929 + Tuple of (frames_array, metadata_dict)

930 + """

931 + cap = cls.open_video_capture(data)

932 +

933 + source = OpenCVVideoBackendMixin.get_video_metadata(cap)

934 +

935 + # recompute source metadata with adjusted duration to ensure correct

936 + # sampling indices computation

937 + target = VideoTargetMetadata(

938 + num_frames=num_frames,

939 + fps=fps,

940 + max_duration=max_duration,

941 )

942

849 - if recovered_map:

850 - logger.info(

851 - "Frame recovery: %d frames recovered using forward scan.",

852 - len(recovered_map),

853 - )

943 + frame_indices_list = cls.compute_frames_index_to_sample(

944 + source=source,

945 + target=target,

946 + )

947

855 - metadata = {

856 - "total_num_frames": total_frames_num,

857 - "fps": original_fps,

858 - "duration": total_duration,

859 - "video_backend": "opencv_dynamic_openpangu",

860 - "frames_indices": valid_frame_indices,

861 - "do_sample_frames": False,

862 - }

948 + frames, valid_frame_indices = cls.read_frames(

949 + cap,

950 + frame_indices_list,

951 + total_frames_num=source.total_frames_num,

952 + frame_recovery=frame_recovery,

953 + )

954 +

955 + # Use transformers transformers.video_utils.VideoMetadata format

956 + metadata = cls.create_hf_metadata(

957 + source=source,

958 + video_backend="opencv_dynamic",

959 + valid_frame_indices=valid_frame_indices,

960 + )

961 return frames, metadata