| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import math |
| 4 | from abc import abstractmethod |
| 5 | from typing import Any, ClassVar, TypeAlias, cast |
| 6 | |
| 7 | import numpy as np |
| 8 | import numpy.typing as npt |
| 9 | import torch |
| 10 | |
| 11 | from vllm.logger import init_logger |
| 12 | from vllm.multimodal.video_decoders import ( |
| 13 | PYNVVIDEOCODEC_VIDEO_BACKEND, |
| 14 | VideoDecoderBackend, |
| 15 | VideoSourceMetadata, |
| 16 | VideoTargetMetadata, |
| 17 | decode_video, |
| 18 | resolve_video_backend_kwargs, |
| 19 | ) |
| 20 | from vllm.utils.import_utils import PlaceholderModule |
| 21 | from vllm.utils.registry import ExtensionManager |
| 22 | |
| 23 | try: |
| 24 | import cv2 |
| 25 | except ImportError: |
| 26 | cv2 = PlaceholderModule("cv2") |
| 27 | |
| 28 | |
| 29 | logger = init_logger(__name__) |
| 30 | |
| 31 | DecodedFrames: TypeAlias = npt.NDArray | torch.Tensor |
| 32 | """Decoded video frames: a host ``np.ndarray``, or a device ``torch.Tensor`` |
| 33 | when a GPU decoding codec is used (e.g. torchcodec with ``device="cuda"``).""" |
| 34 | |
| 35 | |
| 36 | class VideoLoaderRegistry(ExtensionManager): |
| 37 | def __init__(self) -> None: |
| 38 | super().__init__() |
| 39 | self.processor2backend: dict[str, str] = {} |
| 40 | self._requires_gpu: dict[str, bool] = {} |
| 41 | |
| 42 | @staticmethod |
| 43 | def _normalize_registered_video_processors( |
| 44 | video_processor: str | tuple[str, ...] | None, |
| 45 | ) -> tuple[str, ...]: |
| 46 | if video_processor is None: |
| 47 | return () |
| 48 | |
| 49 | if isinstance(video_processor, str): |
| 50 | return (video_processor,) |
| 51 | |
| 52 | if all(isinstance(processor, str) for processor in video_processor): |
| 53 | return video_processor |
| 54 | |
| 55 | raise TypeError( |
| 56 | "video_processor must be a class name or a tuple of class names" |
| 57 | ) |
| 58 | |
| 59 | def register( |
| 60 | self, |
| 61 | name: str, |
| 62 | *, |
| 63 | video_processor: str | tuple[str, ...] | None = None, |
| 64 | requires_gpu: bool = False, |
| 65 | ): |
| 66 | processors = self._normalize_registered_video_processors(video_processor) |
| 67 | |
| 68 | def wrap(cls_to_register): |
| 69 | self.name2class[name] = cls_to_register |
| 70 | self._requires_gpu[name] = requires_gpu |
| 71 | for processor_name in processors: |
| 72 | self.processor2backend[processor_name] = name |
| 73 | return cls_to_register |
| 74 | |
| 75 | return wrap |
| 76 | |
| 77 | def get_backend_for_video_processor( |
| 78 | self, |
| 79 | video_processor: str | None, |
| 80 | ) -> str | None: |
| 81 | if video_processor is None: |
| 82 | return None |
| 83 | |
| 84 | return self.processor2backend.get(video_processor) |
| 85 | |
| 86 | def register_gpu_codec(self, name: str) -> None: |
| 87 | """Mark a codec name as requiring GPU without registering a loader.""" |
| 88 | self._requires_gpu[name] = True |
| 89 | |
| 90 | def backend_requires_gpu(self, name: str) -> bool: |
| 91 | return self._requires_gpu.get(name, False) |
| 92 | |
| 93 | |
| 94 | def get_video_loader_backend_for_processor( |
| 95 | video_processor: str | None, |
| 96 | ) -> str | None: |
| 97 | return VIDEO_LOADER_REGISTRY.get_backend_for_video_processor(video_processor) |
| 98 | |
| 99 | |
| 100 | def resize_video(frames: npt.NDArray, size: tuple[int, int]) -> npt.NDArray: |
| 101 | num_frames, _, _, channels = frames.shape |
| 102 | new_height, new_width = size |
| 103 | resized_frames = np.empty( |
| 104 | (num_frames, new_height, new_width, channels), dtype=frames.dtype |
| 105 | ) |
| 106 | |
| 107 | for i, frame in enumerate(frames): |
| 108 | resized_frame = cv2.resize(frame, (new_width, new_height)) |
| 109 | resized_frames[i] = resized_frame |
| 110 | return resized_frames |
| 111 | |
| 112 | |
| 113 | def rescale_video_size(frames: npt.NDArray, size_factor: float) -> npt.NDArray: |
| 114 | _, height, width, _ = frames.shape |
| 115 | new_height = int(height * size_factor) |
| 116 | new_width = int(width * size_factor) |
| 117 | |
| 118 | return resize_video(frames, (new_height, new_width)) |
| 119 | |
| 120 | |
| 121 | def sample_frames_from_video(frames: npt.NDArray, num_frames: int) -> npt.NDArray: |
| 122 | total_frames = frames.shape[0] |
| 123 | if num_frames == -1: |
| 124 | return frames |
| 125 | |
| 126 | frame_indices = np.linspace(0, total_frames - 1, num_frames, dtype=int) |
| 127 | sampled_frames = frames[frame_indices, ...] |
| 128 | return sampled_frames |
| 129 | |
| 130 | |
| 131 | class VideoLoader: |
| 132 | @classmethod |
| 133 | def compute_frames_index_to_sample( |
| 134 | cls, |
| 135 | source: VideoSourceMetadata, |
| 136 | target: VideoTargetMetadata, |
| 137 | **kwargs, |
| 138 | ) -> list[int]: |
| 139 | """Return the list of frame indices to sample from the video.""" |
| 140 | raise NotImplementedError |
| 141 | |
| 142 | @classmethod |
| 143 | def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata: |
| 144 | """Sampling-algorithm-specific metadata adjustment hook.""" |
| 145 | return source |
| 146 | |
| 147 | @classmethod |
| 148 | def read_frames( |
| 149 | cls, |
| 150 | cap: "cv2.VideoCapture", |
| 151 | frame_idx: list[int], |
| 152 | total_frames_num: int, |
| 153 | *, |
| 154 | frame_recovery: bool = False, |
| 155 | ) -> tuple[npt.NDArray, list[int]]: |
| 156 | from vllm.multimodal.video_decoders.opencv import OpenCVVideoBackendMixin |
| 157 | |
| 158 | return OpenCVVideoBackendMixin.read_frames( |
| 159 | cap, |
| 160 | frame_idx, |
| 161 | total_frames_num, |
| 162 | frame_recovery=frame_recovery, |
| 163 | ) |
| 164 | |
| 165 | @classmethod |
| 166 | @abstractmethod |
| 167 | def load_bytes( |
| 168 | cls, |
| 169 | data: bytes, |
| 170 | **kwargs, |
| 171 | ) -> tuple[DecodedFrames, dict[str, Any]]: |
| 172 | """Load video frames from bytes and return (frames, metadata_dict). |
| 173 | |
| 174 | ``frames`` is a CPU ``np.ndarray`` unless a GPU decoding codec is |
| 175 | used (e.g. torchcodec with ``device="cuda"``), in which case it is |
| 176 | a ``torch.Tensor`` living on that device. |
| 177 | """ |
| 178 | raise NotImplementedError |
| 179 | |
| 180 | @classmethod |
| 181 | def create_hf_metadata( |
| 182 | cls, |
| 183 | source: VideoSourceMetadata, |
| 184 | valid_frame_indices: list[int], |
| 185 | video_backend: str, |
| 186 | ): |
| 187 | return { |
| 188 | "total_num_frames": source.total_frames_num, |
| 189 | "fps": source.original_fps, |
| 190 | "duration": source.duration, |
| 191 | "video_backend": video_backend, |
| 192 | "frames_indices": valid_frame_indices, |
| 193 | "do_sample_frames": len(valid_frame_indices) == source.total_frames_num, |
| 194 | } |
| 195 | |
| 196 | |
| 197 | VIDEO_LOADER_REGISTRY = VideoLoaderRegistry() |
| 198 | VIDEO_LOADER_REGISTRY.register_gpu_codec("deepstream") |
| 199 | |
| 200 | |
| 201 | @VIDEO_LOADER_REGISTRY.register("opencv") |
| 202 | class VideoBackend(VideoLoader): |
| 203 | """Uniform-sampling video backend. |
| 204 | |
| 205 | Samples ``num_frames`` uniformly across the video (or one frame every |
| 206 | ``1/fps`` seconds, whichever produces fewer frames). The decoding codec |
| 207 | is selected via the ``backend`` kwarg (``"opencv"``, ``"torchcodec"``, |
| 208 | ``"pynvvideocodec"``, or ``"deepstream"``), |
| 209 | which can be passed through ``--media-io-kwargs``. Defaults to ``"opencv"``. |
| 210 | """ |
| 211 | |
| 212 | _sampling_suffix: ClassVar[str] = "" |
| 213 | |
| 214 | @classmethod |
| 215 | def compute_frames_index_to_sample( |
| 216 | cls, |
| 217 | source: VideoSourceMetadata, |
| 218 | target: VideoTargetMetadata, |
| 219 | **kwargs, |
| 220 | ) -> list[int]: |
| 221 | total_frames_num = source.total_frames_num |
| 222 | duration = source.duration |
| 223 | num_frames = target.num_frames |
| 224 | fps = target.fps |
| 225 | # resample video to target num_frames and fps |
| 226 | # - the minimum of the two will be used |
| 227 | num_frames_to_sample = total_frames_num |
| 228 | if num_frames > 0: |
| 229 | num_frames_to_sample = min(num_frames, total_frames_num) |
| 230 | if fps > 0: |
| 231 | num_frames_to_sample = min(num_frames_to_sample, math.floor(duration * fps)) |
| 232 | num_frames_to_sample = max(1, num_frames_to_sample) |
| 233 | |
| 234 | if num_frames_to_sample == total_frames_num: |
| 235 | return list(range(num_frames_to_sample)) |
| 236 | return np.linspace( |
| 237 | 0, total_frames_num - 1, num_frames_to_sample, dtype=int |
| 238 | ).tolist() |
| 239 | |
| 240 | @classmethod |
| 241 | def load_bytes( |
| 242 | cls, |
| 243 | data: bytes, |
| 244 | num_frames: int = -1, |
| 245 | fps: int = -1, |
| 246 | max_duration: int = 300, |
| 247 | frame_recovery: bool = False, |
| 248 | *, |
| 249 | backend: VideoDecoderBackend = "opencv", |
| 250 | **kwargs: Any, |
| 251 | ) -> tuple[DecodedFrames, dict[str, Any]]: |
| 252 | """Load sampled frames from raw video bytes. |
| 253 | |
| 254 | Args: |
| 255 | data: Raw video bytes. |
| 256 | num_frames: Target number of frames to sample (``-1`` for all). |
| 257 | fps: Target FPS for sampling (``-1`` for original). |
| 258 | max_duration: Maximum duration in seconds — only used by the |
| 259 | dynamic subclass; ignored here. |
| 260 | frame_recovery: Enable forward-scan recovery for failed frames. |
| 261 | Only honored by the OpenCV codec. |
| 262 | backend: Decoding codec — ``"opencv"``, ``"torchcodec"``, |
| 263 | ``"pynvvideocodec"`` or ``"deepstream"``. |
| 264 | kwargs: Codec-specific options, validated against and forwarded to |
| 265 | ``backend``: |
| 266 | |
| 267 | - ``num_ffmpeg_threads`` (TorchCodec): number of FFmpeg |
| 268 | decoding threads; ``0`` (default) relies on the FFmpeg |
| 269 | default value which is ``min(cpu_count + 1, 16)``. |
| 270 | OpenCV will always use ``min(cpu_count, 16)``. |
| 271 | - ``seek_mode`` (TorchCodec): ``"exact"`` (default) guarantees |
| 272 | frame-accurate sampling by scanning the file on creation, |
| 273 | while ``"approximate"`` skips that scan for faster decoder |
| 274 | creation at the cost of relying on the file's metadata. See |
| 275 | https://meta-pytorch.org/torchcodec/stable/generated_examples/decoding/approximate_mode.html |
| 276 | for details. |
| 277 | - ``device`` (TorchCodec): ``"cpu"`` (default) decodes on the |
| 278 | host and returns a ``np.ndarray``; ``"cuda"`` decodes with |
| 279 | NVDEC and returns the frames as a CUDA ``torch.Tensor``, |
| 280 | which a device-side HF video processor can consume without |
| 281 | a host round-trip. |
| 282 | - ``hw_decoders`` (PyNvVideoCodec): maximum number of |
| 283 | concurrent decoder slots. Defaults to 2 and must be a |
| 284 | positive integer. |
| 285 | - ``pool_size`` / ``timeout_sec`` (DeepStream): decoder pool |
| 286 | size and pool acquisition timeout in seconds. |
| 287 | |
| 288 | Returns: |
| 289 | Tuple of ``(frames, metadata_dict)``, where ``frames`` is a |
| 290 | CPU ``np.ndarray`` unless TorchCodec decodes on ``device="cuda"``. |
| 291 | |
| 292 | """ |
| 293 | target = VideoTargetMetadata( |
| 294 | num_frames=num_frames, fps=fps, max_duration=max_duration |
| 295 | ) |
| 296 | sampling_kwargs, backend_kwargs = resolve_video_backend_kwargs(backend, kwargs) |
| 297 | frames, source, frame_idx, valid = decode_video( |
| 298 | backend, |
| 299 | cls, |
| 300 | data, |
| 301 | target, |
| 302 | sampling_kwargs, |
| 303 | backend_kwargs, |
| 304 | frame_recovery=frame_recovery, |
| 305 | ) |
| 306 | |
| 307 | if len(valid) < len(frame_idx): |
| 308 | logger.warning( |
| 309 | "%s video loading: expected %d frames but got %d.", |
| 310 | backend, |
| 311 | len(frame_idx), |
| 312 | len(valid), |
| 313 | ) |
| 314 | |
| 315 | return frames, cls.create_hf_metadata( |
| 316 | source=source, |
| 317 | video_backend=f"{backend}{cls._sampling_suffix}", |
| 318 | valid_frame_indices=valid, |
| 319 | ) |
| 320 | |
| 321 | |
| 322 | @VIDEO_LOADER_REGISTRY.register(PYNVVIDEOCODEC_VIDEO_BACKEND, requires_gpu=True) |
| 323 | class PyNvVideoCodecVideoBackend(VideoBackend): |
| 324 | """Hardware-accelerated video backend using PyNvVideoCodec. |
| 325 | |
| 326 | The backend first opens the stream only to read metadata and compute the |
| 327 | sampled frame indices. It then acquires the raw decoded RGB byte count from |
| 328 | the process-local multimodal GPU memory pool before decoding the selected |
| 329 | frames into VRAM. Decoded frames are copied into pinned host memory before |
| 330 | the lease is released, so downstream preprocessing continues to receive a |
| 331 | CPU ``np.ndarray`` in NHWC RGB format. |
| 332 | """ |
| 333 | |
| 334 | @classmethod |
| 335 | def load_bytes( |
| 336 | cls, |
| 337 | data: bytes, |
| 338 | num_frames: int = -1, |
| 339 | fps: int = -1, |
| 340 | max_duration: int = 300, |
| 341 | frame_recovery: bool = False, |
| 342 | **kwargs, |
| 343 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 344 | kwargs.pop("backend", None) |
| 345 | return super().load_bytes( |
| 346 | data, |
| 347 | num_frames=num_frames, |
| 348 | fps=fps, |
| 349 | max_duration=max_duration, |
| 350 | frame_recovery=frame_recovery, |
| 351 | backend=PYNVVIDEOCODEC_VIDEO_BACKEND, |
| 352 | **kwargs, |
| 353 | ) |
| 354 | |
| 355 | |
| 356 | @VIDEO_LOADER_REGISTRY.register( |
| 357 | "qwen3_vl", |
| 358 | video_processor=("Qwen3VLVideoProcessor", "Cosmos3EdgeVideoProcessor"), |
| 359 | ) |
| 360 | class Qwen3VLVideoBackend(VideoBackend): |
| 361 | _MAX_FRAMES: ClassVar[int] = 768 |
| 362 | _MAX_FPS: ClassVar[int] = 30 |
| 363 | |
| 364 | @classmethod |
| 365 | def compute_frames_index_to_sample( |
| 366 | cls, |
| 367 | source: VideoSourceMetadata, |
| 368 | target: VideoTargetMetadata, |
| 369 | **kwargs, |
| 370 | ) -> list[int]: |
| 371 | total_frames_num = source.total_frames_num |
| 372 | original_fps = source.original_fps |
| 373 | fps = min(target.fps, cls._MAX_FPS) |
| 374 | max_frame_idx = source.total_frames_num - 1 |
| 375 | min_frames = kwargs.get("min_frames", 4) |
| 376 | max_frames = min(kwargs.get("max_frames", cls._MAX_FRAMES), cls._MAX_FRAMES) |
| 377 | |
| 378 | # Refer to: |
| 379 | # https://github.com/huggingface/transformers/blob/v5.9.0/src/transformers/models/qwen3_vl/video_processing_qwen3_vl.py#L119-L125 |
| 380 | num_frames = int(total_frames_num / original_fps * fps) |
| 381 | num_frames = min(max(num_frames, min_frames), max_frames, total_frames_num) |
| 382 | indices = np.linspace(0, max_frame_idx, num_frames).round().astype(int).tolist() |
| 383 | return indices |
| 384 | |
| 385 | @classmethod |
| 386 | def load_bytes( |
| 387 | cls, |
| 388 | data: bytes, |
| 389 | num_frames: int = -1, |
| 390 | fps: int = 2, |
| 391 | max_duration: int = 300, |
| 392 | frame_recovery: bool = False, |
| 393 | *, |
| 394 | backend: VideoDecoderBackend = "opencv", |
| 395 | **kwargs, |
| 396 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 397 | return super().load_bytes( |
| 398 | data, |
| 399 | num_frames=num_frames, |
| 400 | fps=fps, |
| 401 | max_duration=max_duration, |
| 402 | frame_recovery=frame_recovery, |
| 403 | backend=backend, |
| 404 | **kwargs, |
| 405 | ) |
| 406 | |
| 407 | |
| 408 | @VIDEO_LOADER_REGISTRY.register( |
| 409 | "qwen2_vl", |
| 410 | video_processor="Qwen2VLVideoProcessor", |
| 411 | ) |
| 412 | class Qwen2VLVideoBackend(VideoBackend): |
| 413 | """Qwen2-VL / Qwen2.5-VL fps-based video backend. |
| 414 | |
| 415 | Ports transformers' ``Qwen2VLVideoProcessor.sample_frames`` (fps mode), |
| 416 | shared by Qwen2-VL and Qwen2.5-VL (the latter has no video processor of its |
| 417 | own): sample ``total / original_fps * fps`` frames, clamp to |
| 418 | ``[min_frames, max_frames]`` (4 and 768), floor to a multiple of |
| 419 | ``temporal_patch_size`` (2), and take indices with the exact |
| 420 | ``torch.arange(0, total, total / n)`` call so they match HF byte-for-byte. |
| 421 | |
| 422 | ``num_frames`` is ignored (fps-driven, like the Qwen3-VL loader). The |
| 423 | float32 step can emit an out-of-range tail index (e.g. 451 for a 451-frame |
| 424 | clip); it is clamped to the last valid frame. |
| 425 | """ |
| 426 | |
| 427 | _MAX_FRAMES: ClassVar[int] = 768 |
| 428 | _MAX_FPS: ClassVar[int] = 30 |
| 429 | |
| 430 | @classmethod |
| 431 | def compute_frames_index_to_sample( |
| 432 | cls, |
| 433 | source: VideoSourceMetadata, |
| 434 | target: VideoTargetMetadata, |
| 435 | **kwargs, |
| 436 | ) -> list[int]: |
| 437 | # Refer to: |
| 438 | # https://github.com/huggingface/transformers/blob/v5.7.0/src/transformers/models/qwen2_vl/video_processing_qwen2_vl.py#L122-L190 |
| 439 | total_frames_num = source.total_frames_num |
| 440 | original_fps = source.original_fps |
| 441 | temporal_patch_size = kwargs.get("temporal_patch_size", 2) |
| 442 | min_frames = kwargs.get("min_frames", 4) |
| 443 | max_frames = min(kwargs.get("max_frames", cls._MAX_FRAMES), cls._MAX_FRAMES) |
| 444 | |
| 445 | # vLLM reports original_fps == 0 for clips with unknown/variable fps |
| 446 | # (VFR, malformed, streaming); fail loudly instead of dividing by zero. |
| 447 | if original_fps <= 0: |
| 448 | raise ValueError( |
| 449 | "Qwen2-VL video sampling needs a known source fps, but the " |
| 450 | "container reported 0 (variable or unknown frame rate)." |
| 451 | ) |
| 452 | |
| 453 | max_frames = ( |
| 454 | math.floor(min(max_frames, total_frames_num) / temporal_patch_size) |
| 455 | * temporal_patch_size |
| 456 | ) |
| 457 | n = total_frames_num / original_fps * min(target.fps, cls._MAX_FPS) |
| 458 | n = min(max(n, min_frames), max_frames, total_frames_num) |
| 459 | n = math.floor(n / temporal_patch_size) * temporal_patch_size |
| 460 | |
| 461 | # ``torch.arange`` matches transformers' float32 index math exactly |
| 462 | # (numpy's float64 diverges by a frame on some inputs); clamp the tail |
| 463 | # because that step can emit an index == total_frames_num. |
| 464 | indices = torch.arange(0, total_frames_num, total_frames_num / n).int() |
| 465 | return torch.clamp(indices, max=total_frames_num - 1).tolist() |
| 466 | |
| 467 | @classmethod |
| 468 | def load_bytes( |
| 469 | cls, |
| 470 | data: bytes, |
| 471 | num_frames: int = -1, |
| 472 | fps: int = 2, |
| 473 | max_duration: int = 300, |
| 474 | frame_recovery: bool = False, |
| 475 | *, |
| 476 | backend: VideoDecoderBackend = "opencv", |
| 477 | **kwargs, |
| 478 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 479 | return super().load_bytes( |
| 480 | data, |
| 481 | num_frames=num_frames, |
| 482 | fps=fps, |
| 483 | max_duration=max_duration, |
| 484 | frame_recovery=frame_recovery, |
| 485 | backend=backend, |
| 486 | **kwargs, |
| 487 | ) |
| 488 | |
| 489 | |
| 490 | @VIDEO_LOADER_REGISTRY.register( |
| 491 | "opencv_dynamic", |
| 492 | video_processor="Glm4vVideoProcessor", |
| 493 | ) |
| 494 | class DynamicVideoBackend(VideoBackend): |
| 495 | """Duration-aware dynamic-sampling video backend. |
| 496 | |
| 497 | Samples at ``fps`` up to ``max_duration`` seconds, falling back to |
| 498 | uniform sampling across the full duration when the video is longer |
| 499 | than ``max_duration``. Codec is selectable the same way as |
| 500 | :class:`VideoBackend`. |
| 501 | """ |
| 502 | |
| 503 | _sampling_suffix: ClassVar[str] = "_dynamic" |
| 504 | |
| 505 | @classmethod |
| 506 | def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata: |
| 507 | # Estimate duration from frame count and fps when the container |
| 508 | # does not report it (common for WebM/streaming inputs). |
| 509 | if source.duration: |
| 510 | return source |
| 511 | if source.original_fps > 0: |
| 512 | max_frame_idx = source.total_frames_num - 1 |
| 513 | duration = round(max_frame_idx / source.original_fps) + 1 |
| 514 | else: |
| 515 | duration = 0 |
| 516 | return VideoSourceMetadata( |
| 517 | source.total_frames_num, source.original_fps, duration |
| 518 | ) |
| 519 | |
| 520 | @classmethod |
| 521 | def compute_frames_index_to_sample( |
| 522 | cls, |
| 523 | source: VideoSourceMetadata, |
| 524 | target: VideoTargetMetadata, |
| 525 | **kwargs, |
| 526 | ) -> list[int]: |
| 527 | total_frames_num = source.total_frames_num |
| 528 | duration = source.duration |
| 529 | original_fps = source.original_fps |
| 530 | max_duration = target.max_duration |
| 531 | fps = target.fps |
| 532 | max_frame_idx = source.total_frames_num - 1 |
| 533 | |
| 534 | # Refer to: |
| 535 | # https://github.com/huggingface/transformers/blob/v4.55.4/src/transformers/models/glm4v/video_processing_glm4v.py#L103-L140 |
| 536 | frame_indices_list: list[int] |
| 537 | if duration <= max_duration: |
| 538 | n = int(math.floor(duration * fps)) |
| 539 | frame_indices_list = sorted( |
| 540 | { |
| 541 | min(max_frame_idx, int(math.ceil(i * original_fps / fps))) |
| 542 | for i in range(n) |
| 543 | } |
| 544 | ) |
| 545 | else: |
| 546 | num_samples = int(max_duration * fps) |
| 547 | if num_samples >= total_frames_num: |
| 548 | frame_indices_list = list(range(total_frames_num)) |
| 549 | else: |
| 550 | target_seconds = np.linspace(0, duration, num_samples, endpoint=True) |
| 551 | frame_indices_list = sorted( |
| 552 | { |
| 553 | min(max_frame_idx, int(math.ceil(t * original_fps))) |
| 554 | for t in target_seconds |
| 555 | } |
| 556 | ) |
| 557 | return frame_indices_list |
| 558 | |
| 559 | @classmethod |
| 560 | def load_bytes( |
| 561 | cls, |
| 562 | data: bytes, |
| 563 | num_frames: int = -1, |
| 564 | fps: int = 2, |
| 565 | max_duration: int = 300, |
| 566 | frame_recovery: bool = False, |
| 567 | *, |
| 568 | backend: VideoDecoderBackend = "opencv", |
| 569 | **kwargs, |
| 570 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 571 | return super().load_bytes( |
| 572 | data, |
| 573 | num_frames=num_frames, |
| 574 | fps=fps, |
| 575 | max_duration=max_duration, |
| 576 | frame_recovery=frame_recovery, |
| 577 | backend=backend, |
| 578 | **kwargs, |
| 579 | ) |
| 580 | |
| 581 | |
| 582 | @VIDEO_LOADER_REGISTRY.register( |
| 583 | "glm46v", |
| 584 | video_processor="Glm46VVideoProcessor", |
| 585 | ) |
| 586 | class GLM46VVideoBackend(VideoBackend): |
| 587 | """GLM-4.6V dynamic FPS video backend. |
| 588 | |
| 589 | Faithfully replicates the frame sampling logic from transformers' |
| 590 | ``Glm46VVideoProcessor.sample_frames``: |
| 591 | |
| 592 | - Dynamic FPS thresholds based on effective video duration: |
| 593 | ``{≤30s: 3fps, ≤300s: 1fps, >300s: 0.5fps}`` |
| 594 | - ``temporal_patch_size`` multiplier (default 2) applied to extract count |
| 595 | - Duration capped at 2400s, frame count capped at 640 |
| 596 | - Even frame count enforced (append last frame if odd) |
| 597 | """ |
| 598 | |
| 599 | # Match transformers defaults |
| 600 | _DYNAMIC_FPS_THRESHOLDS: ClassVar[dict[int, float]] = { |
| 601 | 30: 3.0, |
| 602 | 300: 1.0, |
| 603 | 2400: 0.5, |
| 604 | } |
| 605 | _MAX_FRAME_COUNT_DYNAMIC: ClassVar[int] = 640 |
| 606 | _MAX_DURATION: ClassVar[int] = 2400 |
| 607 | |
| 608 | @classmethod |
| 609 | def compute_frames_index_to_sample( |
| 610 | cls, |
| 611 | source: VideoSourceMetadata, |
| 612 | target: VideoTargetMetadata, |
| 613 | **kwargs, |
| 614 | ) -> list[int]: |
| 615 | # Refer to: |
| 616 | # https://github.com/huggingface/transformers/blob/v5.9.0/src/transformers/models/glm46v/video_processing_glm46v.py#L97-L102 |
| 617 | total_frames_num = source.total_frames_num |
| 618 | original_fps = source.original_fps |
| 619 | duration = source.duration |
| 620 | temporal_patch_size = kwargs.get("temporal_patch_size", 2) |
| 621 | |
| 622 | max_frame_idx = total_frames_num - 1 |
| 623 | |
| 624 | # Estimate duration from frame count and fps when not reported |
| 625 | if not duration and original_fps > 0: |
| 626 | duration = round(max_frame_idx / original_fps) + 1 |
| 627 | |
| 628 | effective_duration = min(duration, cls._MAX_DURATION) |
| 629 | |
| 630 | # Select target_fps from dynamic thresholds |
| 631 | if effective_duration <= 30: |
| 632 | target_fps = cls._DYNAMIC_FPS_THRESHOLDS[30] |
| 633 | elif effective_duration <= 300: |
| 634 | target_fps = cls._DYNAMIC_FPS_THRESHOLDS[300] |
| 635 | else: |
| 636 | target_fps = cls._DYNAMIC_FPS_THRESHOLDS[2400] |
| 637 | |
| 638 | extract_t = int(effective_duration * target_fps * temporal_patch_size) |
| 639 | extract_t = min(extract_t, cls._MAX_FRAME_COUNT_DYNAMIC) |
| 640 | |
| 641 | duration_per_frame = 1 / original_fps if original_fps > 0 else 0 |
| 642 | max_second = int(duration) if duration else 0 |
| 643 | |
| 644 | if total_frames_num < extract_t: |
| 645 | frame_indices = np.linspace( |
| 646 | 0, total_frames_num - 1, extract_t, dtype=int |
| 647 | ).tolist() |
| 648 | else: |
| 649 | frame_indices = [] |
| 650 | current_second = 0.0 |
| 651 | inv_fps = 1 / (temporal_patch_size * target_fps) |
| 652 | for frame_index in range(total_frames_num): |
| 653 | if frame_index * duration_per_frame >= current_second: |
| 654 | current_second += inv_fps |
| 655 | frame_indices.append(frame_index) |
| 656 | if current_second >= max_second: |
| 657 | break |
| 658 | |
| 659 | if len(frame_indices) < extract_t: |
| 660 | if len(frame_indices) == 0: |
| 661 | start, end = 0, max(total_frames_num - 1, 0) |
| 662 | else: |
| 663 | start, end = frame_indices[0], frame_indices[-1] |
| 664 | frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist() |
| 665 | elif len(frame_indices) > extract_t: |
| 666 | frame_indices = np.linspace( |
| 667 | 0, total_frames_num - 1, extract_t, dtype=int |
| 668 | ).tolist() |
| 669 | |
| 670 | # Deduplicate |
| 671 | seen: set[int] = set() |
| 672 | uniq: list[int] = [] |
| 673 | for idx in frame_indices: |
| 674 | if idx not in seen: |
| 675 | seen.add(idx) |
| 676 | uniq.append(idx) |
| 677 | |
| 678 | # Ensure even frame count |
| 679 | if len(uniq) & 1: |
| 680 | uniq.append(uniq[-1]) |
| 681 | |
| 682 | return uniq |
| 683 | |
| 684 | @classmethod |
| 685 | def load_bytes( |
| 686 | cls, |
| 687 | data: bytes, |
| 688 | num_frames: int = -1, |
| 689 | fps: int = -1, |
| 690 | max_duration: int = 300, |
| 691 | frame_recovery: bool = False, |
| 692 | *, |
| 693 | backend: VideoDecoderBackend = "opencv", |
| 694 | **kwargs, |
| 695 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 696 | return super().load_bytes( |
| 697 | data, |
| 698 | num_frames=num_frames, |
| 699 | fps=fps, |
| 700 | max_duration=max_duration, |
| 701 | frame_recovery=frame_recovery, |
| 702 | backend=backend, |
| 703 | **kwargs, |
| 704 | ) |
| 705 | |
| 706 | |
| 707 | @VIDEO_LOADER_REGISTRY.register( |
| 708 | "glm5next", |
| 709 | # Both spellings: the borrowed-config type string (``Glm5Next...``) and |
| 710 | # the dedicated transformers classes landing with the new checkpoint |
| 711 | # (``Glm5nextVideoProcessor``, matching ``Glm5nextImageProcessor``). |
| 712 | video_processor=("Glm5NextVideoProcessor", "Glm5nextVideoProcessor"), |
| 713 | ) |
| 714 | class Glm5NextVideoBackend(VideoBackend): |
| 715 | """GLM-5.3-Flash fps-interval video backend. |
| 716 | |
| 717 | Selects frames with the same sampler the processor falls back to, so only |
| 718 | the sampled frames are materialized. ``fps_interval`` semantics (default |
| 719 | 2.0) with a temporal-patch-scaled greedy walk, frame count capped at 2048, |
| 720 | temporal pairs kept even. Request overrides: ``fps`` -> fps interval, |
| 721 | ``max_frames`` -> frame cap, ``temporal_patch_size`` (default 2). |
| 722 | """ |
| 723 | |
| 724 | _SEEK_GAP_THRESHOLD: ClassVar[int] = 64 |
| 725 | _DEFAULT_FPS: ClassVar[float] = 2.0 |
| 726 | _DEFAULT_MAX_FRAMES: ClassVar[int] = 2048 |
| 727 | |
| 728 | @classmethod |
| 729 | def _sample_frame_indices( |
| 730 | cls, |
| 731 | total_frames: int, |
| 732 | fps: float, |
| 733 | duration: float, |
| 734 | *, |
| 735 | target_fps: float | None = None, |
| 736 | max_frame_count: int | None = None, |
| 737 | temporal_patch_size: int = 2, |
| 738 | ) -> list[int]: |
| 739 | """GLM video frame sampling (training-reference parity). |
| 740 | |
| 741 | ``target_fps`` is the ``fps_interval`` request knob. The greedy walk |
| 742 | advances at ``1 / (temporal_patch_size * target_fps)`` seconds, so on |
| 743 | frame-dense sources it collects more candidates than ``extract_t`` and |
| 744 | the ``> extract_t`` fixup re-spreads the picks uniformly with |
| 745 | ``np.linspace`` -- that fallback is the intended reference behavior, not |
| 746 | an accident. Short clips (fewer frames than ``extract_t``) are spread at |
| 747 | evenly spaced timestamps (``floor`` sampling; the linspace variant |
| 748 | samples frames unevenly and cost 4 points on video grounding evals). |
| 749 | Request overrides: ``target_fps`` -> fps interval, ``max_frame_count`` |
| 750 | -> frame cap. |
| 751 | """ |
| 752 | max_frame_idx = total_frames - 1 |
| 753 | if not duration: |
| 754 | duration = (round(max_frame_idx / fps) + 1) if fps else 0 |
| 755 | if max_frame_count is None: |
| 756 | max_frame_count = cls._DEFAULT_MAX_FRAMES |
| 757 | if target_fps is None: |
| 758 | target_fps = cls._DEFAULT_FPS |
| 759 | |
| 760 | extract_t = int(duration * target_fps) |
| 761 | extract_t = min(extract_t, int(max_frame_count)) |
| 762 | |
| 763 | duration_per_frame = 1 / fps |
| 764 | timestamps = [i * duration_per_frame for i in range(total_frames)] |
| 765 | max_second = int(duration) |
| 766 | |
| 767 | if total_frames < extract_t: |
| 768 | frame_indices = [ |
| 769 | math.floor(_i * total_frames / extract_t) for _i in range(extract_t) |
| 770 | ] |
| 771 | else: |
| 772 | frame_indices = [] |
| 773 | current_second = 0.0 |
| 774 | inv_fps = 1 / (temporal_patch_size * target_fps) |
| 775 | for frame_index in range(total_frames): |
| 776 | if timestamps[frame_index] >= current_second: |
| 777 | current_second += inv_fps |
| 778 | frame_indices.append(frame_index) |
| 779 | if current_second >= max_second: |
| 780 | break |
| 781 | |
| 782 | if len(frame_indices) < extract_t: |
| 783 | if len(frame_indices) == 0: |
| 784 | start, end = 0, max(total_frames - 1, 0) |
| 785 | else: |
| 786 | start, end = frame_indices[0], frame_indices[-1] |
| 787 | frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist() |
| 788 | elif len(frame_indices) > extract_t: |
| 789 | frame_indices = np.linspace( |
| 790 | 0, total_frames - 1, extract_t, dtype=int |
| 791 | ).tolist() |
| 792 | |
| 793 | seen, uniq = set(), [] |
| 794 | for idx in frame_indices: |
| 795 | if idx not in seen: |
| 796 | seen.add(idx) |
| 797 | uniq.append(int(idx)) |
| 798 | |
| 799 | if len(uniq) & 1: |
| 800 | uniq.append(uniq[-1]) |
| 801 | |
| 802 | return uniq |
| 803 | |
| 804 | @classmethod |
| 805 | def compute_frames_index_to_sample( |
| 806 | cls, |
| 807 | source: VideoSourceMetadata, |
| 808 | target: VideoTargetMetadata, |
| 809 | **kwargs, |
| 810 | ) -> list[int]: |
| 811 | return cls._sample_frame_indices( |
| 812 | source.total_frames_num, |
| 813 | source.original_fps, |
| 814 | source.duration or 0, |
| 815 | target_fps=target.fps if target.fps > 0 else None, |
| 816 | max_frame_count=kwargs.get("max_frames"), |
| 817 | temporal_patch_size=kwargs.get("temporal_patch_size", 2), |
| 818 | ) |
| 819 | |
| 820 | @classmethod |
| 821 | def read_frames( |
| 822 | cls, |
| 823 | cap: "cv2.VideoCapture", |
| 824 | frame_idx: list[int], |
| 825 | total_frames_num: int, |
| 826 | *, |
| 827 | frame_recovery: bool = False, |
| 828 | ) -> tuple[npt.NDArray, list[int]]: |
| 829 | if frame_recovery: |
| 830 | return super().read_frames( |
| 831 | cap, frame_idx, total_frames_num, frame_recovery=frame_recovery |
| 832 | ) |
| 833 | |
| 834 | wanted = sorted(set(frame_idx)) |
| 835 | width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) |
| 836 | height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) |
| 837 | frames = np.empty((len(wanted), height, width, 3), dtype=np.uint8) |
| 838 | valid_frame_indices: list[int] = [] |
| 839 | current: int | None = None |
| 840 | for target in wanted: |
| 841 | gap = target - current if current is not None else None |
| 842 | if gap is not None and 2 <= gap <= cls._SEEK_GAP_THRESHOLD: |
| 843 | for _ in range(gap - 1): |
| 844 | if not cap.grab(): |
| 845 | current = None |
| 846 | break |
| 847 | else: |
| 848 | current = target - 1 |
| 849 | if current != target - 1: |
| 850 | cap.set(cv2.CAP_PROP_POS_FRAMES, target) |
| 851 | current = target - 1 |
| 852 | ok, frame = cap.read() |
| 853 | if ok: |
| 854 | frames[len(valid_frame_indices)] = cv2.cvtColor( |
| 855 | frame, cv2.COLOR_BGR2RGB |
| 856 | ) |
| 857 | valid_frame_indices.append(target) |
| 858 | current = target |
| 859 | else: |
| 860 | current = None |
| 861 | |
| 862 | valid_num_frames = len(valid_frame_indices) |
| 863 | if valid_num_frames < len(wanted): |
| 864 | logger.warning( |
| 865 | "GLM video loading expected %d sampled frames but only loaded %d.", |
| 866 | len(wanted), |
| 867 | valid_num_frames, |
| 868 | ) |
| 869 | return frames[:valid_num_frames], valid_frame_indices |
| 870 | |
| 871 | |
| 872 | @VIDEO_LOADER_REGISTRY.register( |
| 873 | "glmga", |
| 874 | video_processor="GlmgaVideoProcessor", |
| 875 | ) |
| 876 | class GLMGAVideoBackend(VideoBackend): |
| 877 | _MAX_FRAMES: ClassVar[int] = 640 |
| 878 | _MAX_FPS: ClassVar[int] = 30 |
| 879 | |
| 880 | @classmethod |
| 881 | def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata: |
| 882 | # Estimate duration from frame count and fps when the container |
| 883 | # does not report it (common for WebM/streaming inputs). |
| 884 | if source.duration: |
| 885 | return source |
| 886 | if source.original_fps > 0: |
| 887 | max_frame_idx = source.total_frames_num - 1 |
| 888 | duration = round(max_frame_idx / source.original_fps) + 1 |
| 889 | else: |
| 890 | duration = 0 |
| 891 | return VideoSourceMetadata( |
| 892 | source.total_frames_num, source.original_fps, duration |
| 893 | ) |
| 894 | |
| 895 | @classmethod |
| 896 | def compute_frames_index_to_sample( |
| 897 | cls, |
| 898 | source: VideoSourceMetadata, |
| 899 | target: VideoTargetMetadata, |
| 900 | **kwargs, |
| 901 | ) -> list[int]: |
| 902 | total_frames_num = source.total_frames_num |
| 903 | duration = source.duration |
| 904 | original_fps = source.original_fps |
| 905 | target_fps = min(target.fps, cls._MAX_FPS) |
| 906 | max_frame_idx = source.total_frames_num - 1 |
| 907 | max_frames = min(kwargs.get("max_frames", cls._MAX_FRAMES), cls._MAX_FRAMES) |
| 908 | |
| 909 | duration = duration or round(max_frame_idx / original_fps) + 1 |
| 910 | |
| 911 | extract_t = int(duration * target_fps) |
| 912 | extract_t = min(extract_t, max_frames, total_frames_num) |
| 913 | |
| 914 | duration_per_frame = 1 / original_fps |
| 915 | |
| 916 | if total_frames_num < extract_t: |
| 917 | frame_indices = [ |
| 918 | math.floor(i * total_frames_num / extract_t) for i in range(extract_t) |
| 919 | ] |
| 920 | else: |
| 921 | frame_indices = [] |
| 922 | current_second = 0.0 |
| 923 | inv_fps = 1 / target_fps |
| 924 | for frame_index in range(total_frames_num): |
| 925 | if frame_index * duration_per_frame >= current_second: |
| 926 | current_second += inv_fps |
| 927 | frame_indices.append(frame_index) |
| 928 | if current_second >= duration - inv_fps: |
| 929 | break |
| 930 | |
| 931 | if len(frame_indices) < extract_t: |
| 932 | if len(frame_indices) == 0: |
| 933 | start, end = 0, max(total_frames_num - 1, 0) |
| 934 | else: |
| 935 | start, end = frame_indices[0], frame_indices[-1] |
| 936 | frame_indices = np.linspace(start, end, extract_t, dtype=int).tolist() |
| 937 | elif len(frame_indices) > extract_t: |
| 938 | frame_indices = np.linspace( |
| 939 | 0, total_frames_num - 1, extract_t, dtype=int |
| 940 | ).tolist() |
| 941 | |
| 942 | seen, uniq = set(), [] |
| 943 | for idx in frame_indices: |
| 944 | if idx not in seen: |
| 945 | seen.add(idx) |
| 946 | uniq.append(idx) |
| 947 | |
| 948 | return uniq |
| 949 | |
| 950 | @classmethod |
| 951 | def load_bytes( |
| 952 | cls, |
| 953 | data: bytes, |
| 954 | num_frames: int = -1, |
| 955 | fps: int = 2, |
| 956 | max_duration: int = 300, |
| 957 | frame_recovery: bool = False, |
| 958 | *, |
| 959 | backend: VideoDecoderBackend = "opencv", |
| 960 | **kwargs, |
| 961 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 962 | frames, metadata = super().load_bytes( |
| 963 | data, |
| 964 | num_frames=num_frames, |
| 965 | fps=fps, |
| 966 | max_duration=max_duration, |
| 967 | frame_recovery=frame_recovery, |
| 968 | backend=backend, |
| 969 | **kwargs, |
| 970 | ) |
| 971 | # Ensure even frame count — matches HF's sample_frames even-padding |
| 972 | # and _preprocess temporal_patch_size divisibility check. |
| 973 | if frames.shape[0] & 1: |
| 974 | frames = np.concatenate([frames, frames[-1:]], axis=0) |
| 975 | return frames, metadata |
| 976 | |
| 977 | |
| 978 | @VIDEO_LOADER_REGISTRY.register( |
| 979 | "molmo2", |
| 980 | video_processor="Molmo2VideoProcessor", |
| 981 | ) |
| 982 | class Molmo2VideoBackend(VideoLoader): |
| 983 | @classmethod |
| 984 | def get_candidate_target_fps( |
| 985 | cls, |
| 986 | video_fps: float, |
| 987 | sampling_fps: float, |
| 988 | max_fps: float = 8.0, |
| 989 | ) -> list[float]: |
| 990 | """Return the subset of `video_fps` factors that remain multiples |
| 991 | of `sampling_fps`. |
| 992 | |
| 993 | Examples: |
| 994 | >>> get_candidate_target_fps(video_fps=6, sampling_fps=2) |
| 995 | [2, 6] |
| 996 | >>> get_candidate_target_fps(video_fps=5, sampling_fps=1) |
| 997 | [1, 5] |
| 998 | >>> get_candidate_target_fps(video_fps=2, sampling_fps=2) |
| 999 | [2] |
| 1000 | >>> get_candidate_target_fps(video_fps=5, sampling_fps=2) |
| 1001 | Traceback (most recent call last): |
| 1002 | ... |
| 1003 | ValueError: sampling_fps=2 must divide video_fps=5 to produce |
| 1004 | consistent frame steps. |
| 1005 | |
| 1006 | """ |
| 1007 | if sampling_fps is None: |
| 1008 | raise ValueError("sampling_fps must be provided") |
| 1009 | |
| 1010 | video_fps = int(video_fps) |
| 1011 | sampling_fps = int(sampling_fps) |
| 1012 | max_fps = int(max_fps) |
| 1013 | |
| 1014 | if video_fps <= 0 or sampling_fps <= 0: |
| 1015 | raise ValueError( |
| 1016 | "video_fps and sampling_fps must be positive " |
| 1017 | f"(got {video_fps}, {sampling_fps})" |
| 1018 | ) |
| 1019 | if video_fps % sampling_fps != 0: |
| 1020 | raise ValueError( |
| 1021 | f"sampling_fps={sampling_fps} must divide video_fps={video_fps}." |
| 1022 | ) |
| 1023 | |
| 1024 | candidates = [] |
| 1025 | for candidate in range(sampling_fps, video_fps + 1, sampling_fps): |
| 1026 | if candidate > max_fps: |
| 1027 | break |
| 1028 | if video_fps % candidate == 0: |
| 1029 | candidates.append(float(candidate)) |
| 1030 | |
| 1031 | return candidates |
| 1032 | |
| 1033 | @classmethod |
| 1034 | def get_target_fps( |
| 1035 | cls, |
| 1036 | video_fps: float, |
| 1037 | max_frames: int, |
| 1038 | total_frames: int, |
| 1039 | frame_sample_mode: str, |
| 1040 | candidate_target_fps: list[float], |
| 1041 | ) -> float | None: |
| 1042 | """Get the target fps that best spans the video and samples the most |
| 1043 | frames. |
| 1044 | """ |
| 1045 | num_frames_sampled = 0 |
| 1046 | selected_target_fps = None |
| 1047 | for target_fps in candidate_target_fps: |
| 1048 | step_size = max(int(video_fps / target_fps), 1) |
| 1049 | num_frames_sampled_at_fps = int(total_frames / step_size) |
| 1050 | if num_frames_sampled == 0: |
| 1051 | if ( |
| 1052 | "uniform" in frame_sample_mode |
| 1053 | and num_frames_sampled_at_fps > max_frames |
| 1054 | ): |
| 1055 | break |
| 1056 | selected_target_fps = target_fps |
| 1057 | num_frames_sampled = num_frames_sampled_at_fps |
| 1058 | |
| 1059 | else: |
| 1060 | # the candidate sampling fps increases so frame count can't decrease |
| 1061 | assert num_frames_sampled <= num_frames_sampled_at_fps |
| 1062 | if num_frames_sampled_at_fps > max_frames: |
| 1063 | # choose the sampling fps that spans the video |
| 1064 | continue |
| 1065 | |
| 1066 | elif num_frames_sampled_at_fps > num_frames_sampled: |
| 1067 | # both are less than max_frames; choose the one with higher |
| 1068 | # density of frames sampled |
| 1069 | selected_target_fps = target_fps |
| 1070 | num_frames_sampled = num_frames_sampled_at_fps |
| 1071 | return selected_target_fps |
| 1072 | |
| 1073 | @classmethod |
| 1074 | def get_frame_times_and_chosen_fps( |
| 1075 | cls, |
| 1076 | selected_target_fps: float | None, |
| 1077 | total_frames: int, |
| 1078 | max_frames: int, |
| 1079 | video_fps: float, |
| 1080 | ) -> tuple[float | None, npt.NDArray]: |
| 1081 | if selected_target_fps is None: |
| 1082 | frame_indices = np.linspace( |
| 1083 | 0, total_frames, max_frames, endpoint=False, dtype=int |
| 1084 | ) |
| 1085 | else: |
| 1086 | step_size = max(int(video_fps / selected_target_fps), 1) |
| 1087 | frame_indices = np.arange(0, total_frames, step_size) |
| 1088 | if len(frame_indices) > max_frames: |
| 1089 | frame_indices = frame_indices[:max_frames] |
| 1090 | return selected_target_fps, frame_indices |
| 1091 | |
| 1092 | @classmethod |
| 1093 | def sample_times( |
| 1094 | cls, |
| 1095 | duration: float, |
| 1096 | max_frames: int, |
| 1097 | frame_sample_mode: str, |
| 1098 | max_fps: int | None, |
| 1099 | candidate_target_fps: list[float] | None = None, |
| 1100 | **kwargs, |
| 1101 | ) -> npt.NDArray: |
| 1102 | if frame_sample_mode == "fps": |
| 1103 | assert candidate_target_fps is not None |
| 1104 | # Try larger and larger FPSs until we hit one that can't span the video |
| 1105 | sampling_fps = candidate_target_fps[0] |
| 1106 | for candidate_fps in candidate_target_fps[1:]: |
| 1107 | if max_frames / candidate_fps < duration: |
| 1108 | break |
| 1109 | sampling_fps = candidate_fps |
| 1110 | times = np.arange(0, max_frames) / sampling_fps |
| 1111 | times = times[times < duration] |
| 1112 | return times |
| 1113 | elif frame_sample_mode == "uniform_last_frame": |
| 1114 | if max_fps is not None: |
| 1115 | max_duration = ( |
| 1116 | max_frames - 1 |
| 1117 | ) / max_fps # -1 to include the last frame |
| 1118 | if max_duration < duration: |
| 1119 | times = np.linspace( |
| 1120 | 0, duration, num=max_frames, endpoint=True, dtype=np.float64 |
| 1121 | ) |
| 1122 | else: |
| 1123 | times = np.arange(0.0, stop=duration, step=1 / max_fps) |
| 1124 | times = np.concatenate([times, [duration]], axis=0) |
| 1125 | assert len(times) <= max_frames |
| 1126 | else: |
| 1127 | times = np.linspace( |
| 1128 | 0, duration, num=max_frames, endpoint=True, dtype=np.float64 |
| 1129 | ) |
| 1130 | return times |
| 1131 | else: |
| 1132 | raise NotImplementedError(frame_sample_mode) |
| 1133 | |
| 1134 | @classmethod |
| 1135 | def compute_frames_index_to_sample( |
| 1136 | cls, |
| 1137 | source: VideoSourceMetadata, |
| 1138 | target: VideoTargetMetadata, |
| 1139 | **kwargs, |
| 1140 | ): |
| 1141 | max_fps = kwargs.get("max_fps") |
| 1142 | frame_sample_mode = kwargs.get("frame_sample_mode") |
| 1143 | if frame_sample_mode is None: |
| 1144 | return list(range(0, source.total_frames_num)) |
| 1145 | |
| 1146 | if frame_sample_mode not in {"uniform_last_frame", "fps"}: |
| 1147 | raise NotImplementedError( |
| 1148 | f"Unsupported frame_sample_mode: {frame_sample_mode}" |
| 1149 | ) |
| 1150 | |
| 1151 | duration = source.duration |
| 1152 | video_fps = source.original_fps |
| 1153 | total_num_frames = source.total_frames_num |
| 1154 | num_frames = target.num_frames |
| 1155 | sampling_fps = target.fps |
| 1156 | |
| 1157 | if frame_sample_mode == "uniform_last_frame" and max_fps is not None: |
| 1158 | if total_num_frames <= 2: |
| 1159 | indices = np.arange(total_num_frames).astype(int) |
| 1160 | elif duration > (num_frames - 1) / max_fps: # -1 to include the last frame |
| 1161 | # uniform fallback |
| 1162 | indices = np.linspace( |
| 1163 | 0, |
| 1164 | total_num_frames - 1, |
| 1165 | num=min(num_frames, total_num_frames), |
| 1166 | endpoint=True, |
| 1167 | ).astype(int) |
| 1168 | else: |
| 1169 | float_indices = np.arange( |
| 1170 | 0.0, |
| 1171 | stop=total_num_frames - 1, |
| 1172 | step=float(video_fps / max_fps), |
| 1173 | ) |
| 1174 | if np.round(float_indices[-1]) != total_num_frames - 1: |
| 1175 | float_indices = np.concatenate( |
| 1176 | [float_indices, [total_num_frames - 1]], axis=0 |
| 1177 | ) |
| 1178 | indices = np.round(float_indices).astype(int) |
| 1179 | assert indices[-1] < total_num_frames |
| 1180 | assert len(float_indices) <= num_frames |
| 1181 | elif frame_sample_mode == "uniform_last_frame": |
| 1182 | indices = np.linspace( |
| 1183 | 0, |
| 1184 | total_num_frames - 1, |
| 1185 | num=min(num_frames, total_num_frames), |
| 1186 | endpoint=True, |
| 1187 | ).astype(int) |
| 1188 | elif frame_sample_mode == "fps": |
| 1189 | candidate_target_fps = cls.get_candidate_target_fps(video_fps, sampling_fps) |
| 1190 | selected_target_fps = cls.get_target_fps( |
| 1191 | video_fps, |
| 1192 | num_frames, |
| 1193 | total_num_frames, |
| 1194 | frame_sample_mode, |
| 1195 | candidate_target_fps, |
| 1196 | ) |
| 1197 | _, indices = cls.get_frame_times_and_chosen_fps( |
| 1198 | selected_target_fps, |
| 1199 | total_num_frames, |
| 1200 | num_frames, |
| 1201 | video_fps, |
| 1202 | ) |
| 1203 | return indices.tolist() |
| 1204 | |
| 1205 | @classmethod |
| 1206 | def load_bytes_opencv( |
| 1207 | cls, |
| 1208 | data: bytes, |
| 1209 | frame_sample_mode: str | None = None, |
| 1210 | num_frames: int = -1, |
| 1211 | max_fps: int = 2, |
| 1212 | sampling_fps: int = 2, |
| 1213 | frame_recovery: bool = False, |
| 1214 | **kwargs, |
| 1215 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 1216 | target = VideoTargetMetadata( |
| 1217 | num_frames=num_frames, |
| 1218 | fps=sampling_fps, |
| 1219 | max_duration=-1, |
| 1220 | ) |
| 1221 | frames, source, _, valid_frame_indices = decode_video( |
| 1222 | "opencv", |
| 1223 | cls, |
| 1224 | data, |
| 1225 | target, |
| 1226 | { |
| 1227 | "frame_sample_mode": frame_sample_mode, |
| 1228 | "max_fps": max_fps, |
| 1229 | }, |
| 1230 | {}, |
| 1231 | frame_recovery=frame_recovery, |
| 1232 | ) |
| 1233 | |
| 1234 | metadata = cls.create_hf_metadata( |
| 1235 | source=source, |
| 1236 | video_backend="opencv", |
| 1237 | valid_frame_indices=valid_frame_indices, |
| 1238 | ) |
| 1239 | |
| 1240 | return frames, metadata |
| 1241 | |
| 1242 | @classmethod |
| 1243 | def load_bytes( |
| 1244 | cls, |
| 1245 | data: bytes, |
| 1246 | num_frames: int = -1, |
| 1247 | **kwargs, |
| 1248 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 1249 | frame_sample_mode = cast(str | None, kwargs.pop("frame_sample_mode", None)) |
| 1250 | max_fps = cast(int, kwargs.pop("max_fps", 2)) |
| 1251 | sampling_fps = cast(int, kwargs.pop("sampling_fps", 2)) |
| 1252 | out = cls.load_bytes_opencv( |
| 1253 | data, |
| 1254 | frame_sample_mode, |
| 1255 | num_frames, |
| 1256 | max_fps, |
| 1257 | sampling_fps, |
| 1258 | **kwargs, |
| 1259 | ) |
| 1260 | return out |
| 1261 | |
| 1262 | |
| 1263 | @VIDEO_LOADER_REGISTRY.register("nemotron_vl") |
| 1264 | class NemotronVLVideoBackend(VideoBackend): |
| 1265 | @classmethod |
| 1266 | def load_bytes( |
| 1267 | cls, |
| 1268 | data: bytes, |
| 1269 | num_frames: int = -1, |
| 1270 | fps: int = -1, |
| 1271 | max_duration: int = 300, |
| 1272 | frame_recovery: bool = False, |
| 1273 | *, |
| 1274 | backend: VideoDecoderBackend = "opencv", |
| 1275 | **kwargs, |
| 1276 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 1277 | frames, metadata = super().load_bytes( |
| 1278 | data, |
| 1279 | num_frames=num_frames, |
| 1280 | fps=fps, |
| 1281 | max_duration=max_duration, |
| 1282 | frame_recovery=frame_recovery, |
| 1283 | backend=backend, |
| 1284 | **kwargs, |
| 1285 | ) |
| 1286 | |
| 1287 | metadata = dict(metadata) |
| 1288 | metadata["original_video_bytes"] = data |
| 1289 | |
| 1290 | return frames, metadata |
| 1291 | |
| 1292 | |
| 1293 | @VIDEO_LOADER_REGISTRY.register("openpangu") |
| 1294 | class OpenCVDynamicOpenPanguVideoBackend(VideoLoader): |
| 1295 | @classmethod |
| 1296 | def compute_frames_index_to_sample( |
| 1297 | cls, |
| 1298 | source: VideoSourceMetadata, |
| 1299 | target: VideoTargetMetadata, |
| 1300 | **kwargs, |
| 1301 | ) -> list[int]: |
| 1302 | total_frames_num = source.total_frames_num |
| 1303 | original_fps = source.original_fps |
| 1304 | num_frames = target.num_frames |
| 1305 | fps = target.fps |
| 1306 | |
| 1307 | # The timestamp of the rightmost frame, cannot be used to calculate frame 0. |
| 1308 | if total_frames_num >= 1 and original_fps > 0: |
| 1309 | total_duration = (total_frames_num - 1) / original_fps |
| 1310 | else: |
| 1311 | total_duration = 0 |
| 1312 | |
| 1313 | # `fps` is the FPS parameter passed in for sampling, |
| 1314 | # -1 indicates that sampling can be performed directly without FPS limitation. |
| 1315 | if fps > 0: |
| 1316 | # Num_frames is the maximum number of frames to sample. |
| 1317 | # If fewer frames are sampled at this sample_fps, the update duration will be longer. # noqa: E501 |
| 1318 | if num_frames >= int(total_duration * fps) + 1: |
| 1319 | num_frames = int(total_duration * fps) + 1 |
| 1320 | # Under the new maximum frame rate, the video duration of the rightmost frame, # noqa: E501 |
| 1321 | # cannot be calculated for frame 0. |
| 1322 | total_duration = min(total_duration, (num_frames - 1) / fps) |
| 1323 | elif fps != -1: |
| 1324 | raise ValueError( |
| 1325 | f"requires dataset fps is -1 or greater than 0 but got {fps}" |
| 1326 | ) |
| 1327 | |
| 1328 | sample_frame_timestamps = np.linspace( |
| 1329 | 0, total_duration, num_frames, dtype=float |
| 1330 | ) |
| 1331 | frames_indices = [ |
| 1332 | min(total_frames_num - 1, round(t * original_fps)) |
| 1333 | for t in sample_frame_timestamps |
| 1334 | ] |
| 1335 | return frames_indices |
| 1336 | |
| 1337 | @classmethod |
| 1338 | def load_bytes( |
| 1339 | cls, |
| 1340 | data: bytes, |
| 1341 | num_frames: int = -1, |
| 1342 | fps: int = 2, |
| 1343 | max_duration: int = 300, |
| 1344 | frame_recovery: bool = False, |
| 1345 | **kwargs, |
| 1346 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 1347 | """Load video frames with dynamic sampling based on duration. |
| 1348 | |
| 1349 | Args: |
| 1350 | data: Raw video bytes |
| 1351 | num_frames: Not used in dynamic backend |
| 1352 | fps: Target FPS for sampling (default: 2) |
| 1353 | max_duration: Maximum video duration to process (default: 300s) |
| 1354 | frame_recovery: Enable forward-scan recovery for failed frames |
| 1355 | |
| 1356 | Returns: |
| 1357 | Tuple of (frames_array, metadata_dict) |
| 1358 | |
| 1359 | """ |
| 1360 | # recompute source metadata with adjusted duration to ensure correct |
| 1361 | # sampling indices computation |
| 1362 | target = VideoTargetMetadata( |
| 1363 | num_frames=num_frames, |
| 1364 | fps=fps, |
| 1365 | max_duration=max_duration, |
| 1366 | ) |
| 1367 | |
| 1368 | frames, source, _, valid_frame_indices = decode_video( |
| 1369 | "opencv", |
| 1370 | cls, |
| 1371 | data, |
| 1372 | target, |
| 1373 | {}, |
| 1374 | {}, |
| 1375 | frame_recovery=frame_recovery, |
| 1376 | ) |
| 1377 | |
| 1378 | # Use transformers.video_utils.VideoMetadata format |
| 1379 | metadata = cls.create_hf_metadata( |
| 1380 | source=source, |
| 1381 | video_backend="opencv_dynamic", |
| 1382 | valid_frame_indices=valid_frame_indices, |
| 1383 | ) |
| 1384 | return frames, metadata |