| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from dataclasses import dataclass |
| 5 | from functools import lru_cache |
| 6 | from typing import Any, ClassVar, Literal |
| 7 | |
| 8 | import numpy as np |
| 9 | import numpy.typing as npt |
| 10 | from PIL import Image |
| 11 | |
| 12 | from vllm.multimodal.media.audio import load_audio_pyav |
| 13 | from vllm.transformers_utils.repo_utils import hf_api |
| 14 | |
| 15 | from .base import get_cache_dir |
| 16 | |
| 17 | |
| 18 | def _sample_frame_indices(total_frames: int, num_frames: int) -> npt.NDArray: |
| 19 | return np.linspace(0, total_frames - 1, num_frames, dtype=int) |
| 20 | |
| 21 | |
| 22 | @lru_cache |
| 23 | def download_video_asset(filename: str) -> str: |
| 24 | """Download and open an image from huggingface |
| 25 | repo: raushan-testing-hf/videos-test |
| 26 | """ |
| 27 | video_directory = get_cache_dir() / "video-example-data" |
| 28 | video_directory.mkdir(parents=True, exist_ok=True) |
| 29 | |
| 30 | video_path = video_directory / filename |
| 31 | video_path_str = str(video_path) |
| 32 | if not video_path.exists(): |
| 33 | video_path_str = hf_api().hf_hub_download( |
| 34 | repo_id="raushan-testing-hf/videos-test", |
| 35 | filename=filename, |
| 36 | repo_type="dataset", |
| 37 | cache_dir=video_directory, |
| 38 | ) |
| 39 | return video_path_str |
| 40 | |
| 41 | |
| 42 | def video_to_ndarrays(path: str, num_frames: int = -1) -> npt.NDArray: |
| 43 | import cv2 |
| 44 | |
| 45 | cap = cv2.VideoCapture(path) |
| 46 | if not cap.isOpened(): |
| 47 | raise ValueError(f"Could not open video file {path}") |
| 48 | |
| 49 | total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) |
| 50 | frames = [] |
| 51 | |
| 52 | num_frames = num_frames if num_frames > 0 else total_frames |
| 53 | frame_indices = _sample_frame_indices(total_frames, num_frames) |
| 54 | for idx in range(total_frames): |
| 55 | ok = cap.grab() # next img |
| 56 | if not ok: |
| 57 | break |
| 58 | if idx in frame_indices: # only decompress needed |
| 59 | ret, frame = cap.retrieve() |
| 60 | if ret: |
| 61 | # OpenCV uses BGR format, we need to convert it to RGB |
| 62 | # for PIL and transformers compatibility |
| 63 | frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) |
| 64 | |
| 65 | frames = np.stack(frames) |
| 66 | if len(frames) < num_frames: |
| 67 | raise ValueError( |
| 68 | f"Could not read enough frames from video file {path}" |
| 69 | f" (expected {num_frames} frames, got {len(frames)})" |
| 70 | ) |
| 71 | return frames |
| 72 | |
| 73 | |
| 74 | def video_to_pil_images_list(path: str, num_frames: int = -1) -> list[Image.Image]: |
| 75 | frames = video_to_ndarrays(path, num_frames) |
| 76 | return [Image.fromarray(frame) for frame in frames] |
| 77 | |
| 78 | |
| 79 | def video_get_metadata(path: str, num_frames: int = -1) -> dict[str, Any]: |
| 80 | import cv2 |
| 81 | |
| 82 | cap = cv2.VideoCapture(path) |
| 83 | if not cap.isOpened(): |
| 84 | raise ValueError(f"Could not open video file {path}") |
| 85 | |
| 86 | total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) |
| 87 | fps = cap.get(cv2.CAP_PROP_FPS) |
| 88 | duration = total_frames / fps if fps > 0 else 0 |
| 89 | |
| 90 | if num_frames == -1 or num_frames > total_frames: |
| 91 | num_frames = total_frames |
| 92 | frame_indices = _sample_frame_indices(total_frames, num_frames) |
| 93 | |
| 94 | metadata = { |
| 95 | "total_num_frames": num_frames, |
| 96 | "fps": fps, |
| 97 | "duration": duration, |
| 98 | "video_backend": "opencv", |
| 99 | "frames_indices": frame_indices.tolist(), |
| 100 | # extra field used to control hf processor's video |
| 101 | # sampling behavior |
| 102 | "do_sample_frames": num_frames == total_frames, |
| 103 | } |
| 104 | return metadata |
| 105 | |
| 106 | |
| 107 | VideoAssetName = Literal["baby_reading"] |
| 108 | |
| 109 | |
| 110 | @dataclass(frozen=True) |
| 111 | class VideoAsset: |
| 112 | name: VideoAssetName |
| 113 | num_frames: int = -1 |
| 114 | |
| 115 | _NAME_TO_FILE: ClassVar[dict[VideoAssetName, str]] = { |
| 116 | "baby_reading": "sample_demo_1.mp4", |
| 117 | } |
| 118 | |
| 119 | @property |
| 120 | def filename(self) -> str: |
| 121 | return self._NAME_TO_FILE[self.name] |
| 122 | |
| 123 | @property |
| 124 | def video_path(self) -> str: |
| 125 | return download_video_asset(self.filename) |
| 126 | |
| 127 | @property |
| 128 | def pil_images(self) -> list[Image.Image]: |
| 129 | ret = video_to_pil_images_list(self.video_path, self.num_frames) |
| 130 | return ret |
| 131 | |
| 132 | @property |
| 133 | def np_ndarrays(self) -> npt.NDArray: |
| 134 | ret = video_to_ndarrays(self.video_path, self.num_frames) |
| 135 | return ret |
| 136 | |
| 137 | @property |
| 138 | def metadata(self) -> dict[str, Any]: |
| 139 | ret = video_get_metadata(self.video_path, self.num_frames) |
| 140 | return ret |
| 141 | |
| 142 | def get_audio(self, sampling_rate: float | None = None) -> npt.NDArray: |
| 143 | """Read audio data from the video asset, used in Qwen2.5-Omni examples. |
| 144 | |
| 145 | See also: examples/generate/multimodal/qwen2_5_omni/only_thinker.py |
| 146 | """ |
| 147 | return load_audio_pyav(self.video_path, sr=sampling_rate)[0] |