| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import math |
| 5 | from collections.abc import Mapping, Sequence |
| 6 | from typing import Any, cast |
| 7 | |
| 8 | import numpy.typing as npt |
| 9 | import torch |
| 10 | from transformers import BatchFeature |
| 11 | from transformers.video_utils import VideoMetadata |
| 12 | |
| 13 | from vllm.config.multimodal import ( |
| 14 | MultiModalDummyOptions, |
| 15 | ) |
| 16 | from vllm.inputs import MultiModalDataDict |
| 17 | from vllm.multimodal.inputs import ( |
| 18 | MultiModalFieldConfig, |
| 19 | MultiModalKwargsItems, |
| 20 | ) |
| 21 | from vllm.multimodal.parse import ( |
| 22 | ImageSize, |
| 23 | MultiModalDataItems, |
| 24 | ) |
| 25 | from vllm.multimodal.processing import ( |
| 26 | BaseDummyInputsBuilder, |
| 27 | BaseMultiModalProcessor, |
| 28 | BaseProcessingInfo, |
| 29 | PromptReplacement, |
| 30 | PromptUpdate, |
| 31 | PromptUpdateDetails, |
| 32 | ) |
| 33 | from vllm.multimodal.processing.processor import HFMultiModalInputs |
| 34 | from vllm.multimodal.video import ( |
| 35 | VIDEO_LOADER_REGISTRY, |
| 36 | VideoBackend, |
| 37 | VideoDecoderBackend, |
| 38 | VideoSourceMetadata, |
| 39 | VideoTargetMetadata, |
| 40 | ) |
| 41 | from vllm.transformers_utils.configs.minimax_m3 import MiniMaxM3Config |
| 42 | from vllm.transformers_utils.processors.minimax_m3 import ( |
| 43 | MIN_SHORT_SIDE_PIXEL, |
| 44 | MiniMaxM3VLImageProcessor, |
| 45 | MiniMaxM3VLVideoProcessor, |
| 46 | MiniMaxVLProcessor, |
| 47 | smart_resize, |
| 48 | ) |
| 49 | |
| 50 | # Upper bound on the number of frames used to build the dummy video during |
| 51 | # memory profiling. Sized to the worst-case video the processor accepts: |
| 52 | # ``max_total_pixels // max_pixels_per_frame`` = 301,056,000 // 602,112 = 500 |
| 53 | # frames, each at the video processor's per-frame ``max_pixels`` (768 * 28 * 28 |
| 54 | # = 602,112). This reaches the true worst-case ~192,000 vision tokens, but only |
| 55 | # because the dummy video is sized via ``get_video_size_with_most_features()`` |
| 56 | # (the video ``max_pixels`` bound), not the smaller image bound. Without a cap, |
| 57 | # ``_get_max_video_frames(seq_len)`` with M3's large ``max_model_len`` yields |
| 58 | # ~1400 frames, producing a multi-GB dummy tensor that overflows the |
| 59 | # multimodal encoder cache. |
| 60 | _MAX_FRAMES_PER_VIDEO = 500 |
| 61 | |
| 62 | |
| 63 | class MiniMaxM3VLProcessingInfo(BaseProcessingInfo): |
| 64 | IMAGE_TOKEN = "]<]image[>[" |
| 65 | VIDEO_TOKEN = "]<]video[>[" |
| 66 | VISION_START_TOKEN = "]<]start of image[>[" |
| 67 | VISION_END_TOKEN = "]<]end of image[>[" |
| 68 | |
| 69 | def get_hf_config(self) -> MiniMaxM3Config: |
| 70 | return self.ctx.get_hf_config(MiniMaxM3Config) |
| 71 | |
| 72 | def get_hf_processor(self, **kwargs: object) -> MiniMaxVLProcessor: |
| 73 | # The released checkpoint only ships the processor as remote code |
| 74 | # (via ``auto_map``). Construct the vendored processor directly so the |
| 75 | # model loads without ``--trust-remote-code``. |
| 76 | return self.ctx.get_hf_processor(MiniMaxVLProcessor, **kwargs) |
| 77 | |
| 78 | def get_supported_mm_limits(self) -> Mapping[str, int | None]: |
| 79 | return {"image": None, "video": None} |
| 80 | |
| 81 | def get_mm_max_tokens_per_item( |
| 82 | self, |
| 83 | seq_len: int, |
| 84 | mm_counts: Mapping[str, int], |
| 85 | ) -> Mapping[str, int]: |
| 86 | return { |
| 87 | "image": self.get_max_image_tokens(), |
| 88 | "video": self.get_max_video_tokens(seq_len, mm_counts), |
| 89 | } |
| 90 | |
| 91 | def get_image_processor(self, **kwargs: object) -> MiniMaxM3VLImageProcessor: |
| 92 | return self.get_hf_processor(**kwargs).image_processor |
| 93 | |
| 94 | def get_video_processor(self, **kwargs: object) -> MiniMaxM3VLVideoProcessor: |
| 95 | return self.get_hf_processor(**kwargs).video_processor |
| 96 | |
| 97 | def _get_vision_info( |
| 98 | self, |
| 99 | *, |
| 100 | image_width: int, |
| 101 | image_height: int, |
| 102 | num_frames: int, |
| 103 | image_processor, |
| 104 | ) -> tuple[ImageSize, int]: |
| 105 | """Compute resized image size and number of vision tokens. |
| 106 | |
| 107 | Mirrors the processor's Qwen-style ``smart_resize`` (area bound by |
| 108 | ``max_pixels``) so token counts match the actual processor output. |
| 109 | """ |
| 110 | patch_size: int = image_processor.patch_size |
| 111 | merge_size: int = image_processor.merge_size |
| 112 | temporal_patch_size: int = image_processor.temporal_patch_size |
| 113 | factor = patch_size * merge_size |
| 114 | max_pixels: int = image_processor.max_pixels |
| 115 | # Long-side resize spec (opt-in). ``image_processor`` is the *video* |
| 116 | # processor when counting video tokens, so read the bounds off it. |
| 117 | max_long_side_pixel = getattr(image_processor, "max_long_side_pixel", None) |
| 118 | min_short_side_pixel = getattr( |
| 119 | image_processor, "min_short_side_pixel", MIN_SHORT_SIDE_PIXEL |
| 120 | ) |
| 121 | |
| 122 | new_h, new_w = smart_resize( |
| 123 | image_height, |
| 124 | image_width, |
| 125 | factor=factor, |
| 126 | max_pixels=max_pixels, |
| 127 | max_long_side_pixel=max_long_side_pixel, |
| 128 | min_short_side_pixel=min_short_side_pixel, |
| 129 | # Token counting must not raise; the volumetric/area cap is enforced |
| 130 | # in the processor's _preprocess on the real inputs. |
| 131 | max_total_pixels=None, |
| 132 | ) |
| 133 | grid_h = new_h // patch_size |
| 134 | grid_w = new_w // patch_size |
| 135 | |
| 136 | # Pad frames to be divisible by temporal_patch_size |
| 137 | padded_frames = num_frames + (-num_frames % temporal_patch_size) |
| 138 | grid_t = max(padded_frames // temporal_patch_size, 1) |
| 139 | |
| 140 | num_tokens = grid_t * grid_h * grid_w // (merge_size**2) |
| 141 | return ImageSize(width=new_w, height=new_h), num_tokens |
| 142 | |
| 143 | def get_num_image_tokens( |
| 144 | self, |
| 145 | *, |
| 146 | image_width: int, |
| 147 | image_height: int, |
| 148 | image_processor, |
| 149 | mm_kwargs: Mapping[str, object], |
| 150 | ) -> int: |
| 151 | _, n = self._get_vision_info( |
| 152 | image_width=image_width, |
| 153 | image_height=image_height, |
| 154 | num_frames=1, |
| 155 | image_processor=image_processor, |
| 156 | ) |
| 157 | return n |
| 158 | |
| 159 | def get_num_video_tokens( |
| 160 | self, |
| 161 | *, |
| 162 | image_width: int, |
| 163 | image_height: int, |
| 164 | num_frames: int, |
| 165 | image_processor, |
| 166 | mm_kwargs: Mapping[str, object], |
| 167 | ) -> int: |
| 168 | _, n = self._get_vision_info( |
| 169 | image_width=image_width, |
| 170 | image_height=image_height, |
| 171 | num_frames=num_frames, |
| 172 | image_processor=image_processor, |
| 173 | ) |
| 174 | return n |
| 175 | |
| 176 | def get_image_size_with_most_features(self) -> ImageSize: |
| 177 | # Largest square (a multiple of patch_size*merge_size) whose area is |
| 178 | # within the image processor's bound — this yields the most vision |
| 179 | # tokens for one image. With the long-side spec the square side is |
| 180 | # capped by ``max_long_side_pixel`` (and the fixed ``max_total_pixels``); |
| 181 | # otherwise it is bound by the ``max_pixels`` area. |
| 182 | image_processor = self.get_image_processor() |
| 183 | factor = image_processor.patch_size * image_processor.merge_size |
| 184 | max_long_side_pixel = getattr(image_processor, "max_long_side_pixel", None) |
| 185 | if max_long_side_pixel is not None: |
| 186 | side_px = min( |
| 187 | max_long_side_pixel, |
| 188 | math.isqrt(image_processor.max_total_pixels), |
| 189 | ) |
| 190 | else: |
| 191 | side_px = math.isqrt(image_processor.max_pixels) |
| 192 | side = max(factor, (side_px // factor) * factor) |
| 193 | return ImageSize(width=side, height=side) |
| 194 | |
| 195 | def get_video_size_with_most_features(self) -> ImageSize: |
| 196 | # Per-frame size that yields the most vision tokens, bound by the |
| 197 | # *video* processor's ``max_pixels`` (which differs from the image |
| 198 | # bound). Token count depends only on area, so maximize the area |
| 199 | # achievable with both sides a multiple of patch_size*merge_size rather |
| 200 | # than picking the largest square — a square (e.g. 756x756 for M3's |
| 201 | # 602,112 bound) leaves area on the table, undercounting frames. |
| 202 | video_processor = self.get_video_processor() |
| 203 | factor = video_processor.patch_size * video_processor.merge_size |
| 204 | per_frame_pixels = video_processor.max_pixels |
| 205 | max_long_side_pixel = getattr(video_processor, "max_long_side_pixel", None) |
| 206 | if max_long_side_pixel is not None: |
| 207 | # Long-side spec: a frame's worst case is a square capped by |
| 208 | # ``max_long_side_pixel`` (per-frame area, not the volumetric cap). |
| 209 | per_frame_pixels = min(per_frame_pixels, max_long_side_pixel**2) |
| 210 | units = per_frame_pixels // (factor * factor) # h_u * w_u |
| 211 | h_u = math.isqrt(units) |
| 212 | while units % h_u: |
| 213 | h_u -= 1 |
| 214 | return ImageSize(width=(units // h_u) * factor, height=h_u * factor) |
| 215 | |
| 216 | def get_max_image_tokens(self) -> int: |
| 217 | image_processor = self.get_image_processor() |
| 218 | size = self.get_image_size_with_most_features() |
| 219 | return self.get_num_image_tokens( |
| 220 | image_width=size.width, |
| 221 | image_height=size.height, |
| 222 | image_processor=image_processor, |
| 223 | mm_kwargs={}, |
| 224 | ) |
| 225 | |
| 226 | def _get_max_video_frames(self, max_tokens: int) -> int: |
| 227 | video_processor = self.get_video_processor() |
| 228 | size = self.get_video_size_with_most_features() |
| 229 | num_frames = 1 |
| 230 | while True: |
| 231 | next_n = self.get_num_video_tokens( |
| 232 | image_width=size.width, |
| 233 | image_height=size.height, |
| 234 | num_frames=num_frames + 1, |
| 235 | image_processor=video_processor, |
| 236 | mm_kwargs={}, |
| 237 | ) |
| 238 | if next_n > max_tokens: |
| 239 | break |
| 240 | num_frames += 1 |
| 241 | return num_frames |
| 242 | |
| 243 | def get_num_frames_with_most_features( |
| 244 | self, |
| 245 | seq_len: int, |
| 246 | mm_counts: Mapping[str, int], |
| 247 | max_frames_per_video: int = _MAX_FRAMES_PER_VIDEO, |
| 248 | ) -> int: |
| 249 | max_videos = mm_counts.get("video", 0) |
| 250 | max_total_frames = self._get_max_video_frames(seq_len) |
| 251 | max_frames_per_video = min( |
| 252 | max_total_frames // max(max_videos, 1), max_frames_per_video |
| 253 | ) |
| 254 | return max(max_frames_per_video, 1) |
| 255 | |
| 256 | def get_max_video_tokens( |
| 257 | self, |
| 258 | seq_len: int, |
| 259 | mm_counts: Mapping[str, int], |
| 260 | ) -> int: |
| 261 | video_processor = self.get_video_processor() |
| 262 | size = self.get_video_size_with_most_features() |
| 263 | return self.get_num_video_tokens( |
| 264 | image_width=size.width, |
| 265 | image_height=size.height, |
| 266 | num_frames=self.get_num_frames_with_most_features(seq_len, mm_counts), |
| 267 | image_processor=video_processor, |
| 268 | mm_kwargs={}, |
| 269 | ) |
| 270 | |
| 271 | |
| 272 | class MiniMaxM3VLDummyInputsBuilder(BaseDummyInputsBuilder[MiniMaxM3VLProcessingInfo]): |
| 273 | def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: |
| 274 | num_images = mm_counts.get("image", 0) |
| 275 | num_videos = mm_counts.get("video", 0) |
| 276 | image_token: str = self.info.IMAGE_TOKEN |
| 277 | video_token: str = self.info.VIDEO_TOKEN |
| 278 | return image_token * num_images + video_token * num_videos |
| 279 | |
| 280 | def get_dummy_mm_data( |
| 281 | self, |
| 282 | seq_len: int, |
| 283 | mm_counts: Mapping[str, int], |
| 284 | mm_options: MultiModalDummyOptions, |
| 285 | ) -> MultiModalDataDict: |
| 286 | size = self.info.get_image_size_with_most_features() |
| 287 | video_size = self.info.get_video_size_with_most_features() |
| 288 | num_frames = self.info.get_num_frames_with_most_features(seq_len, mm_counts) |
| 289 | return { |
| 290 | "image": self._get_dummy_images( |
| 291 | width=size.width, |
| 292 | height=size.height, |
| 293 | num_images=mm_counts.get("image", 0), |
| 294 | overrides=mm_options.get("image"), |
| 295 | ), |
| 296 | "video": self._get_dummy_videos( |
| 297 | width=video_size.width, |
| 298 | height=video_size.height, |
| 299 | num_frames=num_frames, |
| 300 | num_videos=mm_counts.get("video", 0), |
| 301 | overrides=mm_options.get("video"), |
| 302 | ), |
| 303 | } |
| 304 | |
| 305 | |
| 306 | class MiniMaxM3VLMultiModalProcessor( |
| 307 | BaseMultiModalProcessor[MiniMaxM3VLProcessingInfo] |
| 308 | ): |
| 309 | def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: |
| 310 | return self.dummy_inputs.get_dummy_text(mm_counts) |
| 311 | |
| 312 | def _get_hf_mm_inputs( |
| 313 | self, |
| 314 | mm_items: MultiModalDataItems, |
| 315 | hf_kwargs: Mapping[str, object], |
| 316 | ) -> HFMultiModalInputs: |
| 317 | hf_inputs = super()._get_hf_mm_inputs(mm_items, hf_kwargs) |
| 318 | hf_data = hf_inputs.hf_data |
| 319 | |
| 320 | # With ``video_needs_metadata=True`` each video arrives as a |
| 321 | # ``(frames, metadata)`` tuple. Split the frames back out and forward the |
| 322 | # metadata as ``VideoMetadata`` so the processor emits timestamps. |
| 323 | videos = cast(list | None, hf_data.get("videos")) |
| 324 | video_metadata: list[VideoMetadata] | None = None |
| 325 | if videos: |
| 326 | frames_only = [] |
| 327 | video_metadata = [] |
| 328 | for item in videos: |
| 329 | if isinstance(item, tuple) and len(item) == 2: |
| 330 | frames, meta = item |
| 331 | else: |
| 332 | frames, meta = item, {} |
| 333 | frames_only.append(frames) |
| 334 | meta = { |
| 335 | k: v for k, v in (meta or {}).items() if k != "do_sample_frames" |
| 336 | } |
| 337 | # VideoMetadata requires total_num_frames; derive it for |
| 338 | # dummy/profiling videos whose metadata omits it. fps and |
| 339 | # frames_indices default to None there → no timestamps, which |
| 340 | # stays consistent with _get_prompt_updates. |
| 341 | meta.setdefault("total_num_frames", len(frames)) |
| 342 | video_metadata.append(VideoMetadata(**meta)) |
| 343 | hf_data["videos"] = frames_only |
| 344 | |
| 345 | if video_metadata is not None: |
| 346 | hf_data["video_metadata"] = video_metadata |
| 347 | |
| 348 | return hf_inputs |
| 349 | |
| 350 | def _call_hf_processor( |
| 351 | self, |
| 352 | hf_data: Mapping[str, object], |
| 353 | hf_kwargs: Mapping[str, object], |
| 354 | ) -> BatchFeature: |
| 355 | # Override the video processor's default for vLLM's raw-frame inputs. |
| 356 | merged = dict(do_resize=True, **hf_kwargs) |
| 357 | return self.info.ctx.call_hf_processor( |
| 358 | self.info.get_hf_processor(**hf_kwargs), |
| 359 | hf_data, |
| 360 | merged, |
| 361 | ) |
| 362 | |
| 363 | def _get_mm_fields_config( |
| 364 | self, |
| 365 | hf_inputs: BatchFeature, |
| 366 | hf_processor_mm_kwargs: Mapping[str, object], |
| 367 | ) -> Mapping[str, MultiModalFieldConfig]: |
| 368 | image_grid_thw = hf_inputs.get("image_grid_thw") |
| 369 | video_grid_thw = hf_inputs.get("video_grid_thw") |
| 370 | |
| 371 | # Total patches per item (grid_t * grid_h * grid_w) |
| 372 | image_grid_sizes = ( |
| 373 | image_grid_thw.prod(-1) |
| 374 | if image_grid_thw is not None |
| 375 | else torch.empty(0, dtype=torch.long) |
| 376 | ) |
| 377 | video_grid_sizes = ( |
| 378 | video_grid_thw.prod(-1) |
| 379 | if video_grid_thw is not None |
| 380 | else torch.empty(0, dtype=torch.long) |
| 381 | ) |
| 382 | |
| 383 | return { |
| 384 | "pixel_values": MultiModalFieldConfig.flat_from_sizes( |
| 385 | "image", image_grid_sizes |
| 386 | ), |
| 387 | "image_grid_thw": MultiModalFieldConfig.batched("image", keep_on_cpu=True), |
| 388 | "pixel_values_videos": MultiModalFieldConfig.flat_from_sizes( |
| 389 | "video", video_grid_sizes |
| 390 | ), |
| 391 | "video_grid_thw": MultiModalFieldConfig.batched("video", keep_on_cpu=True), |
| 392 | } |
| 393 | |
| 394 | def _get_prompt_updates( |
| 395 | self, |
| 396 | mm_items: MultiModalDataItems, |
| 397 | hf_processor_mm_kwargs: Mapping[str, object], |
| 398 | out_mm_kwargs: MultiModalKwargsItems, |
| 399 | ) -> Sequence[PromptUpdate]: |
| 400 | hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) |
| 401 | tokenizer = self.info.get_tokenizer() |
| 402 | vocab = tokenizer.get_vocab() |
| 403 | |
| 404 | image_token_id: int = vocab[self.info.IMAGE_TOKEN] |
| 405 | video_token_id: int = vocab[self.info.VIDEO_TOKEN] |
| 406 | start_token_id: int = vocab[self.info.VISION_START_TOKEN] |
| 407 | end_token_id: int = vocab[self.info.VISION_END_TOKEN] |
| 408 | merge_length: int = hf_processor.image_processor.merge_size**2 |
| 409 | |
| 410 | def get_image_replacement(item_idx: int): |
| 411 | grid_thw: torch.Tensor = out_mm_kwargs["image"][item_idx][ |
| 412 | "image_grid_thw" |
| 413 | ].data |
| 414 | # grid_thw shape: (3,) = [1, grid_h, grid_w] |
| 415 | N = int(grid_thw.prod().item()) // merge_length |
| 416 | full = [start_token_id] + [image_token_id] * N + [end_token_id] |
| 417 | return PromptUpdateDetails.select_token_id(full, image_token_id) |
| 418 | |
| 419 | # Per-video metadata (fps + sampled frame indices) is carried on the |
| 420 | # parsed video items; used to reproduce the HF processor's timestamps. |
| 421 | video_items = mm_items.get("video") |
| 422 | video_metadata = getattr(video_items, "metadata", None) |
| 423 | temporal_patch_size: int = hf_processor.video_processor.temporal_patch_size |
| 424 | |
| 425 | def get_video_replacement(item_idx: int): |
| 426 | grid_thw: torch.Tensor = out_mm_kwargs["video"][item_idx][ |
| 427 | "video_grid_thw" |
| 428 | ].data |
| 429 | # grid_thw shape: (3,) = [grid_t, grid_h, grid_w] |
| 430 | # HF model uses VIDEO_TOKEN (not IMAGE_TOKEN) for video frame content: |
| 431 | # processing_minimax.py L245: replace(placeholder, self.VIDEO_TOKEN) |
| 432 | T = int(grid_thw[0].item()) |
| 433 | M = int(grid_thw[1].item() * grid_thw[2].item()) // merge_length |
| 434 | |
| 435 | # Reproduce the HF processor's per-frame timestamp markers |
| 436 | # (processing_minimax.py: ts = frames_indices[frame*tps] / fps, |
| 437 | # rendered as "]<]X.X seconds[>["). Falls back to no timestamps when |
| 438 | # metadata is unavailable (keeping the replacement aligned with the |
| 439 | # processor output in both cases). |
| 440 | meta = ( |
| 441 | video_metadata[item_idx] |
| 442 | if video_metadata is not None and item_idx < len(video_metadata) |
| 443 | else None |
| 444 | ) |
| 445 | fps = meta.get("fps") if meta else None |
| 446 | frames_indices = meta.get("frames_indices") if meta else None |
| 447 | |
| 448 | full: list[int] = [] |
| 449 | for frame_idx in range(T): |
| 450 | if fps is not None and frames_indices is not None: |
| 451 | idx = min(frame_idx * temporal_patch_size, len(frames_indices) - 1) |
| 452 | ts = frames_indices[idx] / fps |
| 453 | full += tokenizer.encode( |
| 454 | f"]<]{ts:.1f} seconds[>[", add_special_tokens=False |
| 455 | ) |
| 456 | full += [start_token_id] + [video_token_id] * M + [end_token_id] |
| 457 | return PromptUpdateDetails.select_token_id(full, video_token_id) |
| 458 | |
| 459 | return [ |
| 460 | PromptReplacement( |
| 461 | modality="image", |
| 462 | target=[image_token_id], |
| 463 | replacement=get_image_replacement, |
| 464 | ), |
| 465 | PromptReplacement( |
| 466 | modality="video", |
| 467 | target=[video_token_id], |
| 468 | replacement=get_video_replacement, |
| 469 | ), |
| 470 | ] |
| 471 | |
| 472 | |
| 473 | @VIDEO_LOADER_REGISTRY.register( |
| 474 | name="minimax_m3_vl", |
| 475 | video_processor="MiniMaxM3VLVideoProcessor", |
| 476 | ) |
| 477 | class MiniMaxM3VideoBackend(VideoBackend): |
| 478 | @classmethod |
| 479 | def load_bytes( |
| 480 | cls, |
| 481 | data: bytes, |
| 482 | num_frames: int = -1, |
| 483 | fps: int = 1, |
| 484 | max_duration: int = 300, |
| 485 | frame_recovery: bool = False, |
| 486 | *, |
| 487 | backend: VideoDecoderBackend = "opencv", |
| 488 | **kwargs, |
| 489 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 490 | return super().load_bytes( |
| 491 | data, |
| 492 | num_frames=num_frames, |
| 493 | fps=fps, |
| 494 | max_duration=max_duration, |
| 495 | frame_recovery=frame_recovery, |
| 496 | backend=backend, |
| 497 | **kwargs, |
| 498 | ) |
| 499 | |
| 500 | @classmethod |
| 501 | def compute_frames_index_to_sample( |
| 502 | cls, |
| 503 | source: VideoSourceMetadata, |
| 504 | target: VideoTargetMetadata, |
| 505 | **kwargs, |
| 506 | ) -> list[int]: |
| 507 | total_frames = source.total_frames_num |
| 508 | video_fps = source.original_fps |
| 509 | fps = target.fps |
| 510 | if total_frames <= 0 or video_fps <= 0 or fps <= 0: |
| 511 | return [0] if total_frames > 0 else [] |
| 512 | |
| 513 | read_time_interval = 1.0 / fps |
| 514 | eps = 1e-4 |
| 515 | |
| 516 | indices: list[int] = [] |
| 517 | prev_kept_ts = -float("inf") |
| 518 | while True: |
| 519 | if not indices: |
| 520 | target_frame = 0 |
| 521 | else: |
| 522 | target_ts = prev_kept_ts + read_time_interval - eps |
| 523 | target_frame = math.ceil(target_ts * video_fps) |
| 524 | target_frame = max(target_frame, indices[-1] + 1) |
| 525 | if target_frame >= total_frames: |
| 526 | break |
| 527 | indices.append(target_frame) |
| 528 | prev_kept_ts = target_frame / video_fps |
| 529 | if not indices: |
| 530 | indices = [0] |
| 531 | return indices |