| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import itertools |
| 5 | import subprocess |
| 6 | import sys |
| 7 | import threading |
| 8 | from contextlib import ExitStack, contextmanager |
| 9 | from pathlib import Path |
| 10 | from typing import Any |
| 11 | |
| 12 | import numpy as np |
| 13 | import numpy.typing as npt |
| 14 | import pytest |
| 15 | import torch |
| 16 | from transformers import AutoVideoProcessor |
| 17 | from transformers.video_utils import VideoMetadata |
| 18 | |
| 19 | from vllm.assets.base import get_vllm_public_assets |
| 20 | from vllm.models.minimax_m3.common.mm_preprocess import MiniMaxM3VideoBackend |
| 21 | from vllm.multimodal.video import ( |
| 22 | PYNVVIDEOCODEC_VIDEO_BACKEND, |
| 23 | VIDEO_LOADER_REGISTRY, |
| 24 | DynamicVideoBackend, |
| 25 | Glm5NextVideoBackend, |
| 26 | GLM46VVideoBackend, |
| 27 | GLMGAVideoBackend, |
| 28 | Molmo2VideoBackend, |
| 29 | Qwen2VLVideoBackend, |
| 30 | Qwen3VLVideoBackend, |
| 31 | VideoBackend, |
| 32 | VideoLoader, |
| 33 | VideoSourceMetadata, |
| 34 | VideoTargetMetadata, |
| 35 | get_video_loader_backend_for_processor, |
| 36 | ) |
| 37 | from vllm.multimodal.video_decoders import decode_video, resolve_video_backend_kwargs |
| 38 | from vllm.multimodal.video_decoders.pynvvideocodec import ( |
| 39 | PYNVVIDEOCODEC_DECODER_CACHE_SIZE, |
| 40 | PyNvVideoCodecDecoderSlot, |
| 41 | PyNvVideoCodecVideoBackendMixin, |
| 42 | _pynv_decoder_pool, |
| 43 | ) |
| 44 | from vllm.platforms import current_platform |
| 45 | from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config |
| 46 | |
| 47 | from .utils import ( |
| 48 | create_edit_list_trimmed_video, |
| 49 | create_long_gop_video, |
| 50 | create_video_from_image, |
| 51 | ) |
| 52 | |
| 53 | pytestmark = pytest.mark.cpu_test |
| 54 | |
| 55 | ASSETS_DIR = Path(__file__).parent / "assets" |
| 56 | assert ASSETS_DIR.exists() |
| 57 | |
| 58 | NUM_FRAMES = 10 |
| 59 | FAKE_OUTPUT_1 = np.random.rand(NUM_FRAMES, 1280, 720, 3) |
| 60 | FAKE_OUTPUT_2 = np.random.rand(NUM_FRAMES, 1280, 720, 3) |
| 61 | |
| 62 | |
| 63 | @contextmanager |
| 64 | def _fresh_decoder_pool(): |
| 65 | """Reset module-level decoder pool for isolated test runs.""" |
| 66 | pool = _pynv_decoder_pool |
| 67 | old_slots = pool.slots |
| 68 | old_active = pool.active |
| 69 | old_cond = pool.cond |
| 70 | old_max = pool.max_slots |
| 71 | pool.slots = [] |
| 72 | pool.active = 0 |
| 73 | pool.cond = threading.Condition() |
| 74 | pool.max_slots = None |
| 75 | try: |
| 76 | yield pool |
| 77 | finally: |
| 78 | pool.slots = old_slots |
| 79 | pool.active = old_active |
| 80 | pool.cond = old_cond |
| 81 | pool.max_slots = old_max |
| 82 | |
| 83 | |
| 84 | @VIDEO_LOADER_REGISTRY.register("test_video_loader_1") |
| 85 | class TestVideoLoader1(VideoLoader): |
| 86 | @classmethod |
| 87 | def load_bytes( |
| 88 | cls, data: bytes, num_frames: int = -1, **kwargs |
| 89 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 90 | return FAKE_OUTPUT_1, {} |
| 91 | |
| 92 | |
| 93 | @VIDEO_LOADER_REGISTRY.register("test_video_loader_2") |
| 94 | class TestVideoLoader2(VideoLoader): |
| 95 | @classmethod |
| 96 | def load_bytes( |
| 97 | cls, data: bytes, num_frames: int = -1, **kwargs |
| 98 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 99 | return FAKE_OUTPUT_2, {} |
| 100 | |
| 101 | |
| 102 | def test_video_loader_registry(): |
| 103 | custom_loader_1 = VIDEO_LOADER_REGISTRY.load("test_video_loader_1") |
| 104 | output_1, _ = custom_loader_1.load_bytes(b"test") |
| 105 | np.testing.assert_array_equal(output_1, FAKE_OUTPUT_1) |
| 106 | |
| 107 | custom_loader_2 = VIDEO_LOADER_REGISTRY.load("test_video_loader_2") |
| 108 | output_2, _ = custom_loader_2.load_bytes(b"test") |
| 109 | np.testing.assert_array_equal(output_2, FAKE_OUTPUT_2) |
| 110 | |
| 111 | |
| 112 | def test_video_loader_type_doesnt_exist(): |
| 113 | with pytest.raises(AssertionError): |
| 114 | VIDEO_LOADER_REGISTRY.load("non_existing_video_loader") |
| 115 | |
| 116 | |
| 117 | def test_video_decoder_backends_are_lazy_imported(): |
| 118 | code = """ |
| 119 | import sys |
| 120 | import vllm.multimodal.video # noqa: F401 |
| 121 | |
| 122 | backend_modules = { |
| 123 | f"vllm.multimodal.video_decoders.{backend}" |
| 124 | for backend in ("opencv", "torchcodec", "pynvvideocodec", "deepstream") |
| 125 | } |
| 126 | loaded = sorted(backend_modules & sys.modules.keys()) |
| 127 | assert not loaded, loaded |
| 128 | """ |
| 129 | result = subprocess.run( |
| 130 | [sys.executable, "-c", code], |
| 131 | capture_output=True, |
| 132 | text=True, |
| 133 | ) |
| 134 | assert result.returncode == 0, result.stderr |
| 135 | |
| 136 | |
| 137 | @pytest.mark.parametrize( |
| 138 | "backend", |
| 139 | ["opencv", "torchcodec", "pynvvideocodec", "deepstream"], |
| 140 | ) |
| 141 | def test_decode_video_imports_only_selected_backend( |
| 142 | backend: str, |
| 143 | monkeypatch: pytest.MonkeyPatch, |
| 144 | ): |
| 145 | imports = [] |
| 146 | decoded = object() |
| 147 | |
| 148 | def fake_decoder(*args, **kwargs): |
| 149 | return decoded |
| 150 | |
| 151 | class FakeBackendModule: |
| 152 | pass |
| 153 | |
| 154 | setattr(FakeBackendModule, f"decode_{backend}", fake_decoder) |
| 155 | |
| 156 | def fake_import_module(name: str, package: str): |
| 157 | imports.append((name, package)) |
| 158 | return FakeBackendModule |
| 159 | |
| 160 | monkeypatch.setattr( |
| 161 | "vllm.multimodal.video_decoders.import_module", fake_import_module |
| 162 | ) |
| 163 | result = decode_video( |
| 164 | backend, |
| 165 | loader_cls=None, |
| 166 | data=b"", |
| 167 | target=VideoTargetMetadata(-1, -1, 300), |
| 168 | sampling_kwargs={}, |
| 169 | backend_kwargs={}, |
| 170 | frame_recovery=False, |
| 171 | ) |
| 172 | |
| 173 | assert result is decoded |
| 174 | assert imports == [(f".{backend}", "vllm.multimodal.video_decoders")] |
| 175 | |
| 176 | |
| 177 | @pytest.mark.parametrize( |
| 178 | ("backend", "kwargs", "expected_sampling", "expected_backend"), |
| 179 | [ |
| 180 | ( |
| 181 | "torchcodec", |
| 182 | { |
| 183 | "min_frames": 4, |
| 184 | "num_ffmpeg_threads": 2, |
| 185 | "seek_mode": "approximate", |
| 186 | "device": "cuda", |
| 187 | }, |
| 188 | {"min_frames": 4}, |
| 189 | {"num_ffmpeg_threads": 2, "seek_mode": "approximate", "device": "cuda"}, |
| 190 | ), |
| 191 | ( |
| 192 | "deepstream", |
| 193 | {"max_frames": 16, "pool_size": 3, "timeout_sec": 10.0}, |
| 194 | {"max_frames": 16}, |
| 195 | {"pool_size": 3, "timeout_sec": 10.0}, |
| 196 | ), |
| 197 | ], |
| 198 | ) |
| 199 | def test_video_backend_kwargs_are_separated_from_sampling_kwargs( |
| 200 | backend: str, |
| 201 | kwargs: dict, |
| 202 | expected_sampling: dict, |
| 203 | expected_backend: dict, |
| 204 | ): |
| 205 | original_kwargs = dict(kwargs) |
| 206 | sampling_kwargs, backend_kwargs = resolve_video_backend_kwargs(backend, kwargs) |
| 207 | |
| 208 | assert sampling_kwargs == expected_sampling |
| 209 | assert backend_kwargs == expected_backend |
| 210 | assert kwargs == original_kwargs |
| 211 | |
| 212 | |
| 213 | def test_video_backend_rejects_options_for_another_decoder(): |
| 214 | with pytest.raises( |
| 215 | ValueError, match="num_ffmpeg_threads is not supported by the 'opencv' backend" |
| 216 | ): |
| 217 | resolve_video_backend_kwargs("opencv", {"num_ffmpeg_threads": 2}) |
| 218 | |
| 219 | with pytest.raises( |
| 220 | ValueError, match="device is not supported by the 'opencv' backend" |
| 221 | ): |
| 222 | resolve_video_backend_kwargs("opencv", {"device": "cuda"}) |
| 223 | |
| 224 | |
| 225 | @pytest.mark.parametrize( |
| 226 | ("backend", "error"), |
| 227 | [ |
| 228 | ("torchcodec", AssertionError), |
| 229 | (PYNVVIDEOCODEC_VIDEO_BACKEND, ValueError), |
| 230 | ], |
| 231 | ) |
| 232 | def test_video_decoder_spec_validates_frame_recovery( |
| 233 | backend: str, error: type[Exception] |
| 234 | ): |
| 235 | with pytest.raises(error, match="frame_recovery is not supported"): |
| 236 | decode_video( |
| 237 | backend, |
| 238 | loader_cls=None, |
| 239 | data=b"", |
| 240 | target=VideoTargetMetadata(-1, -1, 300), |
| 241 | sampling_kwargs={}, |
| 242 | backend_kwargs={}, |
| 243 | frame_recovery=True, |
| 244 | ) |
| 245 | |
| 246 | |
| 247 | @pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") |
| 248 | def test_pynvvideocodec_backend_accounts_raw_decoded_frames( |
| 249 | monkeypatch: pytest.MonkeyPatch, |
| 250 | ): |
| 251 | decoder_cache_sizes = [] |
| 252 | |
| 253 | class FakeMetadata: |
| 254 | width = 10 |
| 255 | height = 20 |
| 256 | average_fps = 5.0 |
| 257 | duration = 2.0 |
| 258 | |
| 259 | class FakeDecoder: |
| 260 | def __init__(self, *args, **kwargs): |
| 261 | decoder_cache_sizes.append(kwargs["decoder_cache_size"]) |
| 262 | |
| 263 | def __len__(self): |
| 264 | return 10 |
| 265 | |
| 266 | def get_stream_metadata(self): |
| 267 | return FakeMetadata() |
| 268 | |
| 269 | class FakeNvc: |
| 270 | class OutputColorType: |
| 271 | RGB = "rgb" |
| 272 | |
| 273 | SimpleDecoder = FakeDecoder |
| 274 | |
| 275 | class RecordingPool: |
| 276 | def __init__(self): |
| 277 | self.acquired: list[int] = [] |
| 278 | |
| 279 | @contextmanager |
| 280 | def acquire(self, size: int): |
| 281 | self.acquired.append(size) |
| 282 | yield |
| 283 | |
| 284 | def fake_decode(cls, file_path: str, frame_idx: list[int], nvc): |
| 285 | return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8) |
| 286 | |
| 287 | pool = RecordingPool() |
| 288 | monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc) |
| 289 | monkeypatch.setattr( |
| 290 | "vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool |
| 291 | ) |
| 292 | monkeypatch.setattr( |
| 293 | PyNvVideoCodecVideoBackendMixin, |
| 294 | "_decode_to_pinned_host", |
| 295 | classmethod(fake_decode), |
| 296 | ) |
| 297 | |
| 298 | loader = VIDEO_LOADER_REGISTRY.load(PYNVVIDEOCODEC_VIDEO_BACKEND) |
| 299 | frames, metadata = loader.load_bytes(b"fake video", num_frames=4) |
| 300 | |
| 301 | assert frames.shape == (4, 20, 10, 3) |
| 302 | assert pool.acquired == [4 * 20 * 10 * 3] |
| 303 | assert decoder_cache_sizes == [PYNVVIDEOCODEC_DECODER_CACHE_SIZE] |
| 304 | assert metadata["video_backend"] == PYNVVIDEOCODEC_VIDEO_BACKEND |
| 305 | assert metadata["frames_indices"] == [0, 3, 6, 9] |
| 306 | |
| 307 | |
| 308 | @pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") |
| 309 | def test_pynvvideocodec_codec_uses_dynamic_sampling_strategy( |
| 310 | monkeypatch: pytest.MonkeyPatch, |
| 311 | ): |
| 312 | decoded_indices = [] |
| 313 | |
| 314 | class FakeMetadata: |
| 315 | width = 10 |
| 316 | height = 20 |
| 317 | average_fps = 5.0 |
| 318 | duration = 2.0 |
| 319 | |
| 320 | class FakeDecoder: |
| 321 | def __init__(self, *args, **kwargs): |
| 322 | pass |
| 323 | |
| 324 | def __len__(self): |
| 325 | return 10 |
| 326 | |
| 327 | def get_stream_metadata(self): |
| 328 | return FakeMetadata() |
| 329 | |
| 330 | class FakeNvc: |
| 331 | class OutputColorType: |
| 332 | RGB = "rgb" |
| 333 | |
| 334 | SimpleDecoder = FakeDecoder |
| 335 | |
| 336 | class RecordingPool: |
| 337 | def __init__(self): |
| 338 | self.acquired: list[int] = [] |
| 339 | |
| 340 | @contextmanager |
| 341 | def acquire(self, size: int): |
| 342 | self.acquired.append(size) |
| 343 | yield |
| 344 | |
| 345 | def fake_decode(cls, file_path: str, frame_idx: list[int], nvc): |
| 346 | decoded_indices.append(frame_idx) |
| 347 | return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8) |
| 348 | |
| 349 | pool = RecordingPool() |
| 350 | monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc) |
| 351 | monkeypatch.setattr( |
| 352 | "vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool |
| 353 | ) |
| 354 | monkeypatch.setattr( |
| 355 | PyNvVideoCodecVideoBackendMixin, |
| 356 | "_decode_to_pinned_host", |
| 357 | classmethod(fake_decode), |
| 358 | ) |
| 359 | |
| 360 | loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic") |
| 361 | frames, metadata = loader.load_bytes( |
| 362 | b"fake video", |
| 363 | fps=2, |
| 364 | max_duration=1, |
| 365 | backend=PYNVVIDEOCODEC_VIDEO_BACKEND, |
| 366 | ) |
| 367 | |
| 368 | assert frames.shape == (2, 20, 10, 3) |
| 369 | assert decoded_indices == [[0, 9]] |
| 370 | assert pool.acquired == [2 * 20 * 10 * 3] |
| 371 | assert metadata["video_backend"] == f"{PYNVVIDEOCODEC_VIDEO_BACKEND}_dynamic" |
| 372 | assert metadata["frames_indices"] == [0, 9] |
| 373 | |
| 374 | |
| 375 | @pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") |
| 376 | def test_pynvvideocodec_corrupted_videos_raise_value_error(): |
| 377 | valid_video = create_long_gop_video(num_frames=2, width=64, height=64) |
| 378 | corrupted_video = (ASSETS_DIR / "corrupted.mp4").read_bytes() |
| 379 | malformed_video = corrupted_video[:128] |
| 380 | |
| 381 | with _fresh_decoder_pool(): |
| 382 | loader = VIDEO_LOADER_REGISTRY.load(PYNVVIDEOCODEC_VIDEO_BACKEND) |
| 383 | with pytest.raises( |
| 384 | ValueError, |
| 385 | match=r"^Invalid or unsupported video file\.$", |
| 386 | ) as malformed_exc: |
| 387 | loader.load_bytes( |
| 388 | malformed_video, |
| 389 | num_frames=1, |
| 390 | hw_decoders=1, |
| 391 | ) |
| 392 | |
| 393 | assert malformed_exc.value.__cause__ is not None |
| 394 | |
| 395 | with pytest.raises( |
| 396 | ValueError, |
| 397 | match=r"^Invalid or unsupported video file\.$", |
| 398 | ) as exc_info: |
| 399 | loader.load_bytes( |
| 400 | corrupted_video, |
| 401 | num_frames=-1, |
| 402 | hw_decoders=1, |
| 403 | ) |
| 404 | |
| 405 | assert exc_info.value.__cause__ is not None |
| 406 | |
| 407 | frames, _ = loader.load_bytes( |
| 408 | valid_video, |
| 409 | num_frames=1, |
| 410 | hw_decoders=1, |
| 411 | ) |
| 412 | assert frames.shape[0] == 1 |
| 413 | |
| 414 | |
| 415 | @pytest.mark.parametrize("hw_decoders", [1, 3]) |
| 416 | def test_pynvvideocodec_decoder_slots_are_bounded( |
| 417 | monkeypatch: pytest.MonkeyPatch, |
| 418 | hw_decoders: int, |
| 419 | ): |
| 420 | class FakeSlot: |
| 421 | pass |
| 422 | |
| 423 | create_count = 0 |
| 424 | with _fresh_decoder_pool(): |
| 425 | PyNvVideoCodecVideoBackendMixin._configure_decoder_slots(hw_decoders) |
| 426 | |
| 427 | def fake_create_slot(cls): |
| 428 | nonlocal create_count |
| 429 | create_count += 1 |
| 430 | return FakeSlot() |
| 431 | |
| 432 | monkeypatch.setattr( |
| 433 | PyNvVideoCodecVideoBackendMixin, |
| 434 | "_create_decoder_slot", |
| 435 | classmethod(fake_create_slot), |
| 436 | ) |
| 437 | |
| 438 | borrowed = threading.Event() |
| 439 | seen_slots = [] |
| 440 | |
| 441 | with ExitStack() as stack: |
| 442 | retained_slots = [ |
| 443 | stack.enter_context( |
| 444 | PyNvVideoCodecVideoBackendMixin._borrow_decoder_slot() |
| 445 | ) |
| 446 | for _ in range(hw_decoders) |
| 447 | ] |
| 448 | |
| 449 | def borrow_extra_slot(): |
| 450 | with ( |
| 451 | PyNvVideoCodecVideoBackendMixin._borrow_decoder_slot() |
| 452 | ) as extra_slot: |
| 453 | seen_slots.append(extra_slot) |
| 454 | borrowed.set() |
| 455 | |
| 456 | thread = threading.Thread(target=borrow_extra_slot) |
| 457 | thread.start() |
| 458 | assert not borrowed.wait(timeout=0.2) |
| 459 | |
| 460 | assert borrowed.wait(timeout=2.0) |
| 461 | thread.join(timeout=2.0) |
| 462 | assert not thread.is_alive() |
| 463 | |
| 464 | assert seen_slots[0] in retained_slots |
| 465 | assert create_count == hw_decoders |
| 466 | |
| 467 | |
| 468 | def test_pynvvideocodec_decoder_slots_are_configured_once( |
| 469 | monkeypatch: pytest.MonkeyPatch, |
| 470 | ): |
| 471 | monkeypatch.setattr(_pynv_decoder_pool, "max_slots", None) |
| 472 | |
| 473 | PyNvVideoCodecVideoBackendMixin._configure_decoder_slots(2) |
| 474 | PyNvVideoCodecVideoBackendMixin._configure_decoder_slots(2) |
| 475 | |
| 476 | with pytest.raises(RuntimeError, match="already configured as 2, got 3"): |
| 477 | PyNvVideoCodecVideoBackendMixin._configure_decoder_slots(3) |
| 478 | |
| 479 | |
| 480 | def test_pynvvideocodec_failed_rebuild_invalidates_decoder_slot(): |
| 481 | events: list[tuple[str, str]] = [] |
| 482 | |
| 483 | class FakeStream: |
| 484 | cuda_stream = "cuda-stream" |
| 485 | |
| 486 | class FakeDecoder: |
| 487 | poisoned = False |
| 488 | |
| 489 | def reconfigure_decoder(self, file_path: str): |
| 490 | self.poisoned = True |
| 491 | events.append(("reconfigure", file_path)) |
| 492 | raise RuntimeError("reconfigure failed") |
| 493 | |
| 494 | old_decoder = FakeDecoder() |
| 495 | slot = PyNvVideoCodecDecoderSlot(FakeStream()) |
| 496 | slot.decoder = old_decoder |
| 497 | slot.source_path = "valid.mp4" |
| 498 | |
| 499 | class FakeNvc: |
| 500 | class OutputColorType: |
| 501 | RGB = "rgb" |
| 502 | |
| 503 | @staticmethod |
| 504 | def SimpleDecoder(file_path: str, **kwargs): |
| 505 | events.append(("construct", file_path)) |
| 506 | assert slot.decoder is None |
| 507 | assert slot.source_path is None |
| 508 | raise RuntimeError("construct failed") |
| 509 | |
| 510 | pool = _pynv_decoder_pool |
| 511 | old_slots = pool.slots |
| 512 | old_active = pool.active |
| 513 | old_cond = pool.cond |
| 514 | old_max = pool.max_slots |
| 515 | try: |
| 516 | pool.slots = [slot] |
| 517 | pool.active = 1 |
| 518 | pool.cond = threading.Condition() |
| 519 | pool.max_slots = 1 |
| 520 | |
| 521 | with ( |
| 522 | pytest.raises(RuntimeError, match="construct failed"), |
| 523 | PyNvVideoCodecVideoBackendMixin._borrow_decoder_slot() as borrowed, |
| 524 | ): |
| 525 | assert borrowed is slot |
| 526 | borrowed.get_decoder( |
| 527 | "unsupported-8k.mp4", |
| 528 | FakeNvc, |
| 529 | device_index=0, |
| 530 | ) |
| 531 | |
| 532 | assert events == [ |
| 533 | ("reconfigure", "unsupported-8k.mp4"), |
| 534 | ("construct", "unsupported-8k.mp4"), |
| 535 | ] |
| 536 | assert old_decoder.poisoned |
| 537 | assert slot.decoder is None |
| 538 | assert slot.source_path is None |
| 539 | assert pool.slots == [slot] |
| 540 | finally: |
| 541 | pool.slots = old_slots |
| 542 | pool.active = old_active |
| 543 | pool.cond = old_cond |
| 544 | pool.max_slots = old_max |
| 545 | |
| 546 | |
| 547 | @pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") |
| 548 | def test_pynvvideocodec_h200_recovers_after_unsupported_8k(): |
| 549 | import PyNvVideoCodec as nvc |
| 550 | import torch |
| 551 | |
| 552 | if "H200" not in torch.cuda.get_device_name(0): |
| 553 | pytest.skip("Requires H200 NVDEC resolution limits") |
| 554 | |
| 555 | valid_video = create_long_gop_video(num_frames=2, width=64, height=64) |
| 556 | unsupported_video = (ASSETS_DIR / "unsupported_8k_h264.mp4").read_bytes() |
| 557 | |
| 558 | old_slots = _pynv_decoder_pool.slots |
| 559 | old_active = _pynv_decoder_pool.active |
| 560 | old_cond = _pynv_decoder_pool.cond |
| 561 | old_max = _pynv_decoder_pool.max_slots |
| 562 | try: |
| 563 | _pynv_decoder_pool.slots = [] |
| 564 | _pynv_decoder_pool.active = 0 |
| 565 | _pynv_decoder_pool.cond = threading.Condition() |
| 566 | _pynv_decoder_pool.max_slots = None |
| 567 | |
| 568 | loader = VIDEO_LOADER_REGISTRY.load(PYNVVIDEOCODEC_VIDEO_BACKEND) |
| 569 | frames_before, _ = loader.load_bytes( |
| 570 | valid_video, |
| 571 | num_frames=1, |
| 572 | hw_decoders=1, |
| 573 | ) |
| 574 | |
| 575 | with pytest.raises(Exception) as exc_info: |
| 576 | loader.load_bytes( |
| 577 | unsupported_video, |
| 578 | num_frames=1, |
| 579 | hw_decoders=1, |
| 580 | ) |
| 581 | |
| 582 | root_cause = exc_info.value |
| 583 | while root_cause.__cause__ is not None: |
| 584 | root_cause = root_cause.__cause__ |
| 585 | assert isinstance(root_cause, nvc.PyNvVCExceptionUnsupported) |
| 586 | assert "MBCount not supported" in str(root_cause) |
| 587 | |
| 588 | frames_after, _ = loader.load_bytes( |
| 589 | valid_video, |
| 590 | num_frames=1, |
| 591 | hw_decoders=1, |
| 592 | ) |
| 593 | |
| 594 | assert frames_after.shape == frames_before.shape |
| 595 | finally: |
| 596 | for slot in _pynv_decoder_pool.slots: |
| 597 | slot.invalidate() |
| 598 | _pynv_decoder_pool.slots = old_slots |
| 599 | _pynv_decoder_pool.active = old_active |
| 600 | _pynv_decoder_pool.cond = old_cond |
| 601 | _pynv_decoder_pool.max_slots = old_max |
| 602 | |
| 603 | |
| 604 | def test_pynvvideocodec_cross_subclass_shares_single_pool( |
| 605 | monkeypatch: pytest.MonkeyPatch, |
| 606 | ): |
| 607 | """Regression test for GHSA-j682-9xp5-rrf3. |
| 608 | |
| 609 | Multiple subclasses of PyNvVideoCodecVideoBackendMixin must share the |
| 610 | same process-wide decoder slot limit rather than getting independent |
| 611 | counters via ClassVar shadowing. |
| 612 | """ |
| 613 | |
| 614 | class MixinSubclassA(PyNvVideoCodecVideoBackendMixin): |
| 615 | pass |
| 616 | |
| 617 | class MixinSubclassB(PyNvVideoCodecVideoBackendMixin): |
| 618 | pass |
| 619 | |
| 620 | class FakeSlot: |
| 621 | pass |
| 622 | |
| 623 | create_count = 0 |
| 624 | |
| 625 | def fake_create_slot(cls): |
| 626 | nonlocal create_count |
| 627 | create_count += 1 |
| 628 | return FakeSlot() |
| 629 | |
| 630 | with _fresh_decoder_pool() as pool: |
| 631 | pool.max_slots = 2 |
| 632 | |
| 633 | with monkeypatch.context() as m: |
| 634 | m.setattr( |
| 635 | PyNvVideoCodecVideoBackendMixin, |
| 636 | "_create_decoder_slot", |
| 637 | classmethod(fake_create_slot), |
| 638 | ) |
| 639 | with ExitStack() as stack: |
| 640 | stack.enter_context(MixinSubclassA._borrow_decoder_slot()) |
| 641 | stack.enter_context(MixinSubclassB._borrow_decoder_slot()) |
| 642 | assert pool.active == 2 |
| 643 | |
| 644 | blocked = threading.Event() |
| 645 | acquired = threading.Event() |
| 646 | |
| 647 | def try_borrow(): |
| 648 | blocked.set() |
| 649 | with MixinSubclassB._borrow_decoder_slot(): |
| 650 | acquired.set() |
| 651 | |
| 652 | t = threading.Thread(target=try_borrow) |
| 653 | t.start() |
| 654 | blocked.wait(timeout=2.0) |
| 655 | assert not acquired.wait(timeout=0.3) |
| 656 | |
| 657 | assert acquired.wait(timeout=2.0) |
| 658 | t.join(timeout=2.0) |
| 659 | assert not t.is_alive() |
| 660 | |
| 661 | assert create_count == 2 |
| 662 | assert len(pool.slots) == 2 |
| 663 | |
| 664 | |
| 665 | @pytest.mark.parametrize("hw_decoders", [0, -1, 1.5, True, "2"]) |
| 666 | def test_pynvvideocodec_rejects_invalid_hw_decoders(hw_decoders: object): |
| 667 | with pytest.raises(ValueError, match="hw_decoders must be a positive integer"): |
| 668 | VideoBackend.load_bytes( |
| 669 | b"fake video", |
| 670 | backend=PYNVVIDEOCODEC_VIDEO_BACKEND, |
| 671 | hw_decoders=hw_decoders, |
| 672 | ) |
| 673 | |
| 674 | |
| 675 | def test_pynvvideocodec_decoder_slot_retains_simple_decoder(): |
| 676 | events: list[tuple[object, ...]] = [] |
| 677 | |
| 678 | class FakeStream: |
| 679 | cuda_stream = "cuda-stream" |
| 680 | |
| 681 | class FakeDecoder: |
| 682 | def __init__(self, file_path: str, **kwargs): |
| 683 | events.append( |
| 684 | ( |
| 685 | "create", |
| 686 | file_path, |
| 687 | kwargs["gpu_id"], |
| 688 | kwargs["cuda_stream"], |
| 689 | kwargs["decoder_cache_size"], |
| 690 | ) |
| 691 | ) |
| 692 | |
| 693 | def reconfigure_decoder(self, file_path: str): |
| 694 | events.append(("reconfigure", file_path)) |
| 695 | |
| 696 | class FakeNvc: |
| 697 | class OutputColorType: |
| 698 | RGB = "rgb" |
| 699 | |
| 700 | SimpleDecoder = FakeDecoder |
| 701 | |
| 702 | slot = PyNvVideoCodecDecoderSlot(FakeStream()) |
| 703 | |
| 704 | decoder = slot.get_decoder("first.mp4", FakeNvc, device_index=7) |
| 705 | assert slot.get_decoder("first.mp4", FakeNvc, device_index=7) is decoder |
| 706 | assert slot.get_decoder("second.mp4", FakeNvc, device_index=7) is decoder |
| 707 | |
| 708 | assert events == [ |
| 709 | ( |
| 710 | "create", |
| 711 | "first.mp4", |
| 712 | 7, |
| 713 | "cuda-stream", |
| 714 | PYNVVIDEOCODEC_DECODER_CACHE_SIZE, |
| 715 | ), |
| 716 | ("reconfigure", "second.mp4"), |
| 717 | ] |
| 718 | assert slot.source_path == "second.mp4" |
| 719 | |
| 720 | |
| 721 | # ============================================================================ |
| 722 | # Video Processor → Video Loader Tests (via model repo) |
| 723 | # ============================================================================ |
| 724 | |
| 725 | |
| 726 | def test_cosmos3_edge_uses_qwen3_vl_video_backend(): |
| 727 | backend = get_video_loader_backend_for_processor("Cosmos3EdgeVideoProcessor") |
| 728 | |
| 729 | assert backend == "qwen3_vl" |
| 730 | assert isinstance(VIDEO_LOADER_REGISTRY.load(backend), Qwen3VLVideoBackend) |
| 731 | |
| 732 | |
| 733 | @pytest.mark.parametrize( |
| 734 | "model_repo, expected_loader_cls, hf_sample_kwargs", |
| 735 | [ |
| 736 | pytest.param( |
| 737 | "allenai/Molmo2-4B", |
| 738 | Molmo2VideoBackend, |
| 739 | None, |
| 740 | marks=pytest.mark.skip( |
| 741 | reason="Video processor not aligned, investigate later.", |
| 742 | ), |
| 743 | id="molmo2", |
| 744 | ), |
| 745 | pytest.param( |
| 746 | "zai-org/GLM-4.1V-9B-Thinking", |
| 747 | DynamicVideoBackend, |
| 748 | None, |
| 749 | id="glm4v", |
| 750 | ), |
| 751 | pytest.param( |
| 752 | "zai-org/GLM-4.6V-Flash", |
| 753 | GLM46VVideoBackend, |
| 754 | None, |
| 755 | id="glm46v", |
| 756 | ), |
| 757 | pytest.param( |
| 758 | "Qwen/Qwen3-VL-4B-Instruct", |
| 759 | Qwen3VLVideoBackend, |
| 760 | None, |
| 761 | id="qwen3vl", |
| 762 | ), |
| 763 | # Qwen2-VL/Qwen2.5-VL ship no ``video_processor_type`` in their |
| 764 | # preprocessor config, so resolution relies on the model_type -> |
| 765 | # video processor fallback in get_video_processor_cls_name_from_config. |
| 766 | # They also ship no default fps/num_frames, so the HF sampler needs an |
| 767 | # explicit target rate; pass fps=2 to match the loader default. |
| 768 | pytest.param( |
| 769 | "Qwen/Qwen2-VL-7B-Instruct", |
| 770 | Qwen2VLVideoBackend, |
| 771 | {"fps": 2}, |
| 772 | id="qwen2vl", |
| 773 | ), |
| 774 | pytest.param( |
| 775 | "Qwen/Qwen2.5-VL-7B-Instruct", |
| 776 | Qwen2VLVideoBackend, |
| 777 | {"fps": 2}, |
| 778 | id="qwen2_5_vl", |
| 779 | ), |
| 780 | pytest.param( |
| 781 | "MiniMaxAI/MiniMax-M3", |
| 782 | MiniMaxM3VideoBackend, |
| 783 | None, |
| 784 | id="minimax_m3_vl", |
| 785 | ), |
| 786 | ], |
| 787 | ) |
| 788 | def test_video_processor_from_model_repo( |
| 789 | model_repo: str, |
| 790 | expected_loader_cls: type, |
| 791 | hf_sample_kwargs: dict[str, int | float] | None, |
| 792 | ): |
| 793 | """Test that a model repo resolves to the correct video loader backend. |
| 794 | |
| 795 | The test downloads the preprocessor config from HuggingFace Hub, |
| 796 | extracts the ``video_processor_type`` field, and verifies it maps |
| 797 | to the expected backend and loader class. When a corresponding HF |
| 798 | ``VideoProcessor.sample_frames`` implementation exists, the test |
| 799 | also verifies that the vLLM backend produces identical frame indices. |
| 800 | """ |
| 801 | video_processor = get_video_processor_cls_name_from_config(model_repo) |
| 802 | assert video_processor is not None, ( |
| 803 | f"Model repo {model_repo!r} did not contain a video_processor_type " |
| 804 | f"in its preprocessor config" |
| 805 | ) |
| 806 | |
| 807 | backend = get_video_loader_backend_for_processor(video_processor) |
| 808 | assert backend is not None |
| 809 | loader = VIDEO_LOADER_REGISTRY.load(backend) |
| 810 | assert isinstance(loader, expected_loader_cls), ( |
| 811 | f"{model_repo!r}: backend={backend!r} loaded " |
| 812 | f"{type(loader)}, expected {expected_loader_cls}" |
| 813 | ) |
| 814 | |
| 815 | # --- Alignment check with HF VideoProcessor.sample_frames --- |
| 816 | processor = AutoVideoProcessor.from_pretrained(model_repo, trust_remote_code=True) |
| 817 | |
| 818 | fps_list = [1, 2, 30, 60] |
| 819 | duration_list = [10, 60, 600] |
| 820 | for fps, duration_secs in itertools.product(fps_list, duration_list): |
| 821 | num_frames = fps * duration_secs |
| 822 | video_bytes = create_long_gop_video( |
| 823 | num_frames=num_frames, |
| 824 | fps=fps, |
| 825 | width=8, |
| 826 | height=8, |
| 827 | ) |
| 828 | |
| 829 | _, vllm_meta = loader.load_bytes(video_bytes) # type: ignore[attr-defined] |
| 830 | |
| 831 | hf_metadata = VideoMetadata( |
| 832 | total_num_frames=vllm_meta["total_num_frames"], |
| 833 | fps=vllm_meta["fps"], |
| 834 | duration=vllm_meta["duration"], |
| 835 | ) |
| 836 | hf_indices = processor.sample_frames(hf_metadata, **(hf_sample_kwargs or {})) |
| 837 | vllm_indices = np.array(vllm_meta["frames_indices"]) |
| 838 | np.testing.assert_array_equal( |
| 839 | hf_indices, |
| 840 | vllm_indices, |
| 841 | err_msg=( |
| 842 | f"{model_repo!r} fps={fps} duration={duration_secs}s: " |
| 843 | f"HF has {len(hf_indices)} indices " |
| 844 | f"{hf_indices[:5].tolist()}..{hf_indices[-5:].tolist()}, " |
| 845 | f"vLLM has {len(vllm_indices)} indices " |
| 846 | f"{vllm_indices[:5].tolist()}..{vllm_indices[-5:].tolist()}" |
| 847 | ), |
| 848 | ) |
| 849 | |
| 850 | |
| 851 | def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch): |
| 852 | """Regression test for handling videos with broken frames. |
| 853 | This test uses a pre-corrupted video file (assets/corrupted.mp4) that |
| 854 | contains broken frames to verify the video loader handles |
| 855 | them gracefully without crashing and returns accurate metadata. |
| 856 | """ |
| 857 | with monkeypatch.context() as m: |
| 858 | m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv") |
| 859 | |
| 860 | # Load the pre-corrupted video file that contains broken frames |
| 861 | corrupted_video_path = ASSETS_DIR / "corrupted.mp4" |
| 862 | |
| 863 | with open(corrupted_video_path, "rb") as f: |
| 864 | video_data = f.read() |
| 865 | |
| 866 | loader = VIDEO_LOADER_REGISTRY.load("opencv") |
| 867 | frames, metadata = loader.load_bytes( |
| 868 | video_data, num_frames=-1, backend="opencv" |
| 869 | ) |
| 870 | |
| 871 | # Verify metadata consistency: |
| 872 | # frames_indices must match actual loaded frames |
| 873 | assert frames.shape[0] == len(metadata["frames_indices"]), ( |
| 874 | f"Frames array size must equal frames_indices length. " |
| 875 | f"Got {frames.shape[0]} frames but " |
| 876 | f"{len(metadata['frames_indices'])} indices" |
| 877 | ) |
| 878 | |
| 879 | # Verify that broken frames were skipped: |
| 880 | # loaded frames should be less than total |
| 881 | assert frames.shape[0] < metadata["total_num_frames"], ( |
| 882 | f"Should load fewer frames than total due to broken frames. " |
| 883 | f"Expected fewer than {metadata['total_num_frames']} frames, " |
| 884 | f"but loaded {frames.shape[0]} frames" |
| 885 | ) |
| 886 | |
| 887 | |
| 888 | def test_video_backend_handles_edit_list_trimmed_video( |
| 889 | monkeypatch: pytest.MonkeyPatch, |
| 890 | ): |
| 891 | """An mp4 edit list (e.g. from a lossless ``ffmpeg -ss ... -c copy`` cut) |
| 892 | hides the decode lead-in: the header still counts every physical sample |
| 893 | while sequential decode only yields the visible frames. Sampling over the |
| 894 | header count used to collapse such videos to the few indices below the |
| 895 | visible count (a single frame for the Qwen loaders) — the loader must |
| 896 | resample over the true stream length instead. |
| 897 | """ |
| 898 | with monkeypatch.context() as m: |
| 899 | m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv") |
| 900 | |
| 901 | video_data, num_visible = create_edit_list_trimmed_video( |
| 902 | num_frames=90, trim_start_frame=60 |
| 903 | ) |
| 904 | |
| 905 | loader = VIDEO_LOADER_REGISTRY.load("opencv") |
| 906 | for backend in ["opencv"]: |
| 907 | frames, metadata = loader.load_bytes( |
| 908 | video_data, num_frames=-1, backend=backend |
| 909 | ) |
| 910 | assert metadata["total_num_frames"] == num_visible, backend |
| 911 | assert frames.shape[0] == num_visible, backend |
| 912 | assert len(metadata["frames_indices"]) == num_visible, backend |
| 913 | # The green channel encodes the source frame index: the visible |
| 914 | # frames must be the trailing ones (60..89), not the lead-in. |
| 915 | mean_green = frames[..., 1].reshape(frames.shape[0], -1).mean(axis=1) |
| 916 | assert abs(mean_green[0] - 60) <= 5, backend |
| 917 | assert abs(mean_green[-1] - 89) <= 5, backend |
| 918 | |
| 919 | # The Qwen samplers used to degenerate to a single frame here. |
| 920 | qwen_frames, qwen_metadata = Qwen2VLVideoBackend.load_bytes(video_data) |
| 921 | assert qwen_metadata["total_num_frames"] == num_visible |
| 922 | assert qwen_frames.shape[0] >= 4 |
| 923 | |
| 924 | |
| 925 | # ============================================================================ |
| 926 | # Frame Recovery Tests |
| 927 | # ============================================================================ |
| 928 | |
| 929 | |
| 930 | def test_video_recovery_simulated_failures(monkeypatch: pytest.MonkeyPatch): |
| 931 | """Test that frame recovery correctly uses the next valid frame when |
| 932 | target frames fail to load. |
| 933 | |
| 934 | Uses corrupted.mp4 and mocks VideoCapture.grab() to fail on specific |
| 935 | frame indices (in addition to the real corruption at frame 17), then |
| 936 | verifies recovery produces more frames. |
| 937 | """ |
| 938 | import cv2 |
| 939 | |
| 940 | with monkeypatch.context() as m: |
| 941 | m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv") |
| 942 | |
| 943 | # Load corrupted.mp4 (26 frames, frame 17 is genuinely corrupted) |
| 944 | video_path = ASSETS_DIR / "corrupted.mp4" |
| 945 | with open(video_path, "rb") as f: |
| 946 | video_data = f.read() |
| 947 | |
| 948 | # Simulate additional failures on frames 3 and 10 |
| 949 | # (in addition to the real corruption at frame 17) |
| 950 | fail_on_frames = {3, 10} |
| 951 | |
| 952 | # Store original VideoCapture class |
| 953 | original_video_capture = cv2.VideoCapture |
| 954 | |
| 955 | class MockVideoCapture: |
| 956 | """Wrapper that simulates grab() failures on specific frames.""" |
| 957 | |
| 958 | def __init__(self, *args, **kwargs): |
| 959 | self._cap = original_video_capture(*args, **kwargs) |
| 960 | self._current_frame = -1 |
| 961 | |
| 962 | def grab(self): |
| 963 | self._current_frame += 1 |
| 964 | if self._current_frame in fail_on_frames: |
| 965 | return False # Simulate failure |
| 966 | return self._cap.grab() |
| 967 | |
| 968 | def retrieve(self): |
| 969 | return self._cap.retrieve() |
| 970 | |
| 971 | def get(self, prop): |
| 972 | return self._cap.get(prop) |
| 973 | |
| 974 | def set(self, prop, value): |
| 975 | # get_video_metadata probes the stream end with a seek; |
| 976 | # keep the simulated frame counter aligned with rewinds. |
| 977 | result = self._cap.set(prop, value) |
| 978 | if prop == cv2.CAP_PROP_POS_FRAMES: |
| 979 | self._current_frame = int(value) - 1 |
| 980 | return result |
| 981 | |
| 982 | def isOpened(self): |
| 983 | return self._cap.isOpened() |
| 984 | |
| 985 | def release(self): |
| 986 | return self._cap.release() |
| 987 | |
| 988 | # Patch cv2.VideoCapture |
| 989 | m.setattr(cv2, "VideoCapture", MockVideoCapture) |
| 990 | |
| 991 | loader = VIDEO_LOADER_REGISTRY.load("opencv") |
| 992 | |
| 993 | # Use num_frames=8 which samples: [0, 3, 7, 10, 14, 17, 21, 25] |
| 994 | # Frame 3: mocked failure, recovery window [3, 7) -> use frame 4 |
| 995 | # Frame 10: mocked failure, recovery window [10, 14) -> use frame 11 |
| 996 | # Frame 17: real corruption, recovery window [17, 21) -> use frame 18 |
| 997 | |
| 998 | # Test WITHOUT recovery - should have fewer frames due to failures |
| 999 | frames_no_recovery, meta_no = loader.load_bytes( |
| 1000 | video_data, num_frames=8, frame_recovery=False, backend="opencv" |
| 1001 | ) |
| 1002 | |
| 1003 | # Test WITH recovery - should recover using next valid frames |
| 1004 | frames_with_recovery, meta_yes = loader.load_bytes( |
| 1005 | video_data, num_frames=8, frame_recovery=True, backend="opencv" |
| 1006 | ) |
| 1007 | |
| 1008 | # With recovery should have MORE frames than without |
| 1009 | # Without: 5 frames (3, 10, 17 all fail) |
| 1010 | # With: 8 frames (all recovered) |
| 1011 | assert frames_with_recovery.shape[0] > frames_no_recovery.shape[0], ( |
| 1012 | f"Recovery should produce more frames. " |
| 1013 | f"Without: {frames_no_recovery.shape[0]}, " |
| 1014 | f"With: {frames_with_recovery.shape[0]}" |
| 1015 | ) |
| 1016 | |
| 1017 | # Verify metadata consistency |
| 1018 | assert frames_no_recovery.shape[0] == len(meta_no["frames_indices"]) |
| 1019 | assert frames_with_recovery.shape[0] == len(meta_yes["frames_indices"]) |
| 1020 | |
| 1021 | # Verify temporal order is preserved |
| 1022 | assert meta_yes["frames_indices"] == sorted(meta_yes["frames_indices"]) |
| 1023 | |
| 1024 | |
| 1025 | def test_video_recovery_with_corrupted_file(monkeypatch: pytest.MonkeyPatch): |
| 1026 | """Test frame recovery with an actual corrupted video file using sparse sampling. |
| 1027 | |
| 1028 | This test uses corrupted.mp4 which has genuine H.264 codec errors on |
| 1029 | frame 17. With num_frames=8, the target frames are [0, 3, 7, 10, 14, 17, 21, 25]. |
| 1030 | Frame 17 is corrupted but frames 18-20 are readable, so recovery can use |
| 1031 | frame 18 to fill in for the failed frame 17. |
| 1032 | |
| 1033 | This test verifies: |
| 1034 | 1. Without recovery: frame 17 is skipped (7 frames loaded) |
| 1035 | 2. With recovery: frame 18 fills in for frame 17 (8 frames loaded) |
| 1036 | 3. Recovery produces MORE frames than without recovery |
| 1037 | 4. Metadata is consistent with loaded frames |
| 1038 | """ |
| 1039 | with monkeypatch.context() as m: |
| 1040 | m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv") |
| 1041 | |
| 1042 | corrupted_video_path = ASSETS_DIR / "corrupted.mp4" |
| 1043 | |
| 1044 | with open(corrupted_video_path, "rb") as f: |
| 1045 | video_data = f.read() |
| 1046 | |
| 1047 | loader = VIDEO_LOADER_REGISTRY.load("opencv") |
| 1048 | |
| 1049 | # Use num_frames=8 which makes frame 17 a target with recovery window [17, 21) |
| 1050 | # Target frames: [0, 3, 7, 10, 14, 17, 21, 25] |
| 1051 | # Frame 17 is corrupted, but frames 18-20 are readable for recovery |
| 1052 | |
| 1053 | # Test without recovery - frame 17 will be skipped |
| 1054 | frames_no_recovery, meta_no_recovery = loader.load_bytes( |
| 1055 | video_data, num_frames=8, frame_recovery=False, backend="opencv" |
| 1056 | ) |
| 1057 | |
| 1058 | # Test with recovery - frame 18 should fill in for frame 17 |
| 1059 | frames_with_recovery, meta_with_recovery = loader.load_bytes( |
| 1060 | video_data, num_frames=8, frame_recovery=True, backend="opencv" |
| 1061 | ) |
| 1062 | |
| 1063 | # Verify metadata consistency for both modes |
| 1064 | assert frames_no_recovery.shape[0] == len(meta_no_recovery["frames_indices"]), ( |
| 1065 | "Frame count must match indices without recovery" |
| 1066 | ) |
| 1067 | assert frames_with_recovery.shape[0] == len( |
| 1068 | meta_with_recovery["frames_indices"] |
| 1069 | ), "Frame count must match indices with recovery" |
| 1070 | |
| 1071 | # KEY ASSERTION: Recovery should produce MORE frames than without recovery |
| 1072 | # Without recovery: 7 frames (frame 17 skipped) |
| 1073 | # With recovery: 8 frames (frame 18 used for frame 17) |
| 1074 | assert frames_with_recovery.shape[0] > frames_no_recovery.shape[0], ( |
| 1075 | f"Recovery should produce more frames with sparse sampling. " |
| 1076 | f"Got {frames_with_recovery.shape[0]} with recovery vs " |
| 1077 | f"{frames_no_recovery.shape[0]} without" |
| 1078 | ) |
| 1079 | |
| 1080 | # Verify we got all 8 requested frames with recovery |
| 1081 | assert frames_with_recovery.shape[0] == 8, ( |
| 1082 | f"With recovery, should load all 8 requested frames. " |
| 1083 | f"Got {frames_with_recovery.shape[0]}" |
| 1084 | ) |
| 1085 | |
| 1086 | # Verify the video metadata is correct |
| 1087 | expected_total_frames = 26 |
| 1088 | assert meta_with_recovery["total_num_frames"] == expected_total_frames, ( |
| 1089 | f"Expected {expected_total_frames} total frames in metadata" |
| 1090 | ) |
| 1091 | |
| 1092 | |
| 1093 | def test_video_recovery_dynamic_backend(monkeypatch: pytest.MonkeyPatch): |
| 1094 | """Test that frame_recovery works with the dynamic video backend. |
| 1095 | |
| 1096 | The dynamic backend samples frames based on fps/duration rather than |
| 1097 | loading all frames. This test verifies recovery works in that context. |
| 1098 | """ |
| 1099 | with monkeypatch.context() as m: |
| 1100 | m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic") |
| 1101 | |
| 1102 | corrupted_video_path = ASSETS_DIR / "corrupted.mp4" |
| 1103 | |
| 1104 | with open(corrupted_video_path, "rb") as f: |
| 1105 | video_data = f.read() |
| 1106 | |
| 1107 | loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic") |
| 1108 | |
| 1109 | # Test without recovery |
| 1110 | frames_no_recovery, meta_no = loader.load_bytes( |
| 1111 | video_data, |
| 1112 | fps=2, |
| 1113 | max_duration=10, |
| 1114 | frame_recovery=False, |
| 1115 | backend="opencv", |
| 1116 | ) |
| 1117 | |
| 1118 | # Test with frame_recovery enabled |
| 1119 | frames_with_recovery, meta_with = loader.load_bytes( |
| 1120 | video_data, fps=2, max_duration=10, frame_recovery=True, backend="opencv" |
| 1121 | ) |
| 1122 | |
| 1123 | # Verify basic properties |
| 1124 | assert frames_no_recovery.shape[0] > 0, ( |
| 1125 | "Should load some frames without recovery" |
| 1126 | ) |
| 1127 | assert frames_with_recovery.shape[0] > 0, ( |
| 1128 | "Should load some frames with recovery" |
| 1129 | ) |
| 1130 | assert "do_sample_frames" in meta_with |
| 1131 | assert meta_with["do_sample_frames"] is False # Dynamic backend always False |
| 1132 | assert frames_with_recovery.shape[0] == len(meta_with["frames_indices"]) |
| 1133 | |
| 1134 | # Key assertion: recovery should help when corrupted frames are sampled |
| 1135 | # We expect recovery to produce >= frames than without recovery |
| 1136 | assert frames_with_recovery.shape[0] >= frames_no_recovery.shape[0], ( |
| 1137 | f"Recovery should produce at least as many frames. " |
| 1138 | f"Got {frames_with_recovery.shape[0]} with recovery vs " |
| 1139 | f"{frames_no_recovery.shape[0]} without" |
| 1140 | ) |
| 1141 | |
| 1142 | |
| 1143 | @pytest.fixture |
| 1144 | def dummy_video_path(tmp_path): |
| 1145 | image_path = get_vllm_public_assets( |
| 1146 | filename="stop_sign.jpg", s3_prefix="vision_model_images" |
| 1147 | ) |
| 1148 | |
| 1149 | video_path = tmp_path / "test_RGB_video.mp4" |
| 1150 | create_video_from_image(str(image_path), str(video_path), num_frames=1800, fps=30) |
| 1151 | return video_path |
| 1152 | |
| 1153 | |
| 1154 | # ============================================================================ |
| 1155 | # TorchCodec Backend Tests |
| 1156 | # ============================================================================ |
| 1157 | |
| 1158 | |
| 1159 | def test_torchcodec_backend_loads_frames( |
| 1160 | dummy_video_path, monkeypatch: pytest.MonkeyPatch |
| 1161 | ): |
| 1162 | """Test that the torchcodec codec backend can load frames.""" |
| 1163 | pytest.importorskip("torchcodec") |
| 1164 | with monkeypatch.context() as m: |
| 1165 | m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv") |
| 1166 | |
| 1167 | with open(dummy_video_path, "rb") as f: |
| 1168 | video_data = f.read() |
| 1169 | |
| 1170 | loader = VIDEO_LOADER_REGISTRY.load("opencv") |
| 1171 | frames, metadata = loader.load_bytes( |
| 1172 | video_data, num_frames=8, backend="torchcodec" |
| 1173 | ) |
| 1174 | |
| 1175 | assert frames.ndim == 4 |
| 1176 | assert frames.shape[3] == 3 # RGB |
| 1177 | assert frames.shape[0] == 8 |
| 1178 | assert frames.shape[0] == len(metadata["frames_indices"]) |
| 1179 | assert metadata["video_backend"] == "torchcodec" |
| 1180 | assert "total_num_frames" in metadata |
| 1181 | assert "fps" in metadata |
| 1182 | assert "duration" in metadata |
| 1183 | |
| 1184 | |
| 1185 | def test_torchcodec_dynamic_backend_loads_frames( |
| 1186 | dummy_video_path, monkeypatch: pytest.MonkeyPatch |
| 1187 | ): |
| 1188 | """Test that the torchcodec codec with dynamic sampling can load frames.""" |
| 1189 | pytest.importorskip("torchcodec") |
| 1190 | with monkeypatch.context() as m: |
| 1191 | m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic") |
| 1192 | |
| 1193 | with open(dummy_video_path, "rb") as f: |
| 1194 | video_data = f.read() |
| 1195 | |
| 1196 | loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic") |
| 1197 | frames, metadata = loader.load_bytes( |
| 1198 | video_data, fps=2, max_duration=10, backend="torchcodec" |
| 1199 | ) |
| 1200 | |
| 1201 | assert frames.ndim == 4 |
| 1202 | assert frames.shape[3] == 3 # RGB |
| 1203 | assert frames.shape[0] > 0 |
| 1204 | assert frames.shape[0] == len(metadata["frames_indices"]) |
| 1205 | assert metadata["video_backend"] == "torchcodec_dynamic" |
| 1206 | |
| 1207 | |
| 1208 | def test_torchcodec_backend_rejects_frame_recovery(dummy_video_path): |
| 1209 | """frame_recovery is OpenCV-only; torchcodec must reject it.""" |
| 1210 | pytest.importorskip("torchcodec") |
| 1211 | with open(dummy_video_path, "rb") as f: |
| 1212 | video_data = f.read() |
| 1213 | |
| 1214 | loader = VIDEO_LOADER_REGISTRY.load("opencv") |
| 1215 | with pytest.raises(AssertionError): |
| 1216 | loader.load_bytes( |
| 1217 | video_data, num_frames=8, backend="torchcodec", frame_recovery=True |
| 1218 | ) |
| 1219 | |
| 1220 | |
| 1221 | @pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") |
| 1222 | def test_torchcodec_backend_cuda_decodes_on_gpu(dummy_video_path): |
| 1223 | """With device="cuda", torchcodec decodes via NVDEC and keeps the frames |
| 1224 | on the GPU instead of returning a host-side numpy array.""" |
| 1225 | pytest.importorskip("torchcodec") |
| 1226 | |
| 1227 | with open(dummy_video_path, "rb") as f: |
| 1228 | video_data = f.read() |
| 1229 | |
| 1230 | loader = VIDEO_LOADER_REGISTRY.load("opencv") |
| 1231 | frames, metadata = loader.load_bytes( |
| 1232 | video_data, num_frames=8, backend="torchcodec", device="cuda" |
| 1233 | ) |
| 1234 | |
| 1235 | assert isinstance(frames, torch.Tensor) |
| 1236 | assert frames.device.type == "cuda" |
| 1237 | assert frames.dtype == torch.uint8 |
| 1238 | assert frames.ndim == 4 |
| 1239 | assert frames.shape[3] == 3 # RGB |
| 1240 | assert frames.shape[0] == 8 |
| 1241 | assert frames.shape[0] == len(metadata["frames_indices"]) |
| 1242 | assert metadata["video_backend"] == "torchcodec" |
| 1243 | |
| 1244 | |
| 1245 | def test_torchcodec_backend_returns_target_frames_not_keyframes(): |
| 1246 | """Regression test: torchcodec must return the requested frames, not the |
| 1247 | GOP keyframe they seek back to. |
| 1248 | |
| 1249 | A long GOP (single keyframe at frame 0) with a per-frame green-channel |
| 1250 | marker. With ``seek_mode="exact"`` torchcodec resolves each index to the |
| 1251 | exact frame, so the returned markers must be distinct, ordered, and match |
| 1252 | the requested indices. |
| 1253 | """ |
| 1254 | pytest.importorskip("torchcodec") |
| 1255 | num_frames = 50 |
| 1256 | num_sampled = 4 |
| 1257 | height, width = 64, 64 |
| 1258 | |
| 1259 | video_bytes = create_long_gop_video( |
| 1260 | num_frames=num_frames, width=width, height=height |
| 1261 | ) |
| 1262 | |
| 1263 | loader = VIDEO_LOADER_REGISTRY.load("opencv") |
| 1264 | frames, metadata = loader.load_bytes( |
| 1265 | video_bytes, num_frames=num_sampled, backend="torchcodec" |
| 1266 | ) |
| 1267 | assert frames.shape == (num_sampled, height, width, 3) |
| 1268 | |
| 1269 | requested = list(metadata["frames_indices"]) |
| 1270 | assert len(requested) == num_sampled |
| 1271 | |
| 1272 | actual = [int(f[height // 2, width // 2, 1]) for f in frames] |
| 1273 | |
| 1274 | assert len(set(actual)) == num_sampled, ( |
| 1275 | f"torchcodec returned only {len(set(actual))} distinct frames for " |
| 1276 | f"{num_sampled} requested indices: markers={actual}, " |
| 1277 | f"requested={requested}. Keyframe-snap regression." |
| 1278 | ) |
| 1279 | |
| 1280 | assert actual == sorted(actual), f"Returned frames out of order: markers={actual}" |
| 1281 | |
| 1282 | for marker, want_idx in zip(actual, requested): |
| 1283 | assert abs(marker - want_idx) <= 10, ( |
| 1284 | f"Frame mismatch: requested index {want_idx}, " |
| 1285 | f"got marker {marker} (tolerance ±10)" |
| 1286 | ) |
| 1287 | |
| 1288 | |
| 1289 | @pytest.mark.parametrize( |
| 1290 | "loader_key, kwargs, expected_num_frames", |
| 1291 | [ |
| 1292 | # uniform sampling + opencv codec |
| 1293 | pytest.param( |
| 1294 | "opencv", |
| 1295 | {"num_frames": 32, "backend": "opencv"}, |
| 1296 | 32, |
| 1297 | id="opencv-num_frames", |
| 1298 | ), |
| 1299 | pytest.param("opencv", {"fps": 2, "backend": "opencv"}, 120, id="opencv-fps"), |
| 1300 | pytest.param( |
| 1301 | "opencv", |
| 1302 | {"num_frames": 500, "fps": 2, "backend": "opencv"}, |
| 1303 | 120, |
| 1304 | id="opencv-num_frames_wins_fps", |
| 1305 | ), |
| 1306 | # dynamic sampling + opencv codec |
| 1307 | pytest.param( |
| 1308 | "opencv_dynamic", |
| 1309 | {"fps": 1, "max_duration": 60, "backend": "opencv"}, |
| 1310 | 60, |
| 1311 | id="opencv_dynamic-within_max_duration", |
| 1312 | ), |
| 1313 | pytest.param( |
| 1314 | "opencv_dynamic", |
| 1315 | {"fps": 2, "max_duration": 30, "backend": "opencv"}, |
| 1316 | 60, |
| 1317 | id="opencv_dynamic-exceeds_max_duration", |
| 1318 | ), |
| 1319 | pytest.param( |
| 1320 | "openpangu", {"num_frames": 32, "fps": -1}, 32, id="openpangu-num_frames" |
| 1321 | ), |
| 1322 | pytest.param( |
| 1323 | "molmo2", |
| 1324 | {"num_frames": 32, "frame_sample_mode": "uniform_last_frame"}, |
| 1325 | 32, |
| 1326 | id="molmo2-uniform_last_frame", |
| 1327 | ), |
| 1328 | pytest.param( |
| 1329 | "molmo2", |
| 1330 | {"fps": 2, "frame_sample_mode": "fps"}, |
| 1331 | 119, |
| 1332 | id="molmo2-fps", |
| 1333 | ), |
| 1334 | # glm46v dynamic FPS (1800 frames @ 30fps = 60s) |
| 1335 | # 60s falls in (30, 300] → target_fps=1.0, extract_t = 60*1.0*2 = 120 |
| 1336 | pytest.param( |
| 1337 | "glm46v", |
| 1338 | {"backend": "opencv"}, |
| 1339 | 120, |
| 1340 | id="glm46v-60s", |
| 1341 | ), |
| 1342 | # uniform sampling + torchcodec codec (same frame counts as opencv) |
| 1343 | pytest.param( |
| 1344 | "opencv", |
| 1345 | {"num_frames": 32, "backend": "torchcodec"}, |
| 1346 | 32, |
| 1347 | id="torchcodec-num_frames", |
| 1348 | ), |
| 1349 | pytest.param( |
| 1350 | "opencv", {"fps": 2, "backend": "torchcodec"}, 120, id="torchcodec-fps" |
| 1351 | ), |
| 1352 | pytest.param( |
| 1353 | "opencv", |
| 1354 | {"num_frames": 500, "fps": 2, "backend": "torchcodec"}, |
| 1355 | 120, |
| 1356 | id="torchcodec-num_frames_wins_fps", |
| 1357 | ), |
| 1358 | # dynamic sampling + torchcodec codec |
| 1359 | pytest.param( |
| 1360 | "opencv_dynamic", |
| 1361 | {"fps": 1, "max_duration": 60, "backend": "torchcodec"}, |
| 1362 | 60, |
| 1363 | id="torchcodec_dynamic-within_max_duration", |
| 1364 | ), |
| 1365 | pytest.param( |
| 1366 | "opencv_dynamic", |
| 1367 | {"fps": 2, "max_duration": 30, "backend": "torchcodec"}, |
| 1368 | 60, |
| 1369 | id="torchcodec_dynamic-exceeds_max_duration", |
| 1370 | ), |
| 1371 | # glm46v dynamic FPS + torchcodec codec |
| 1372 | pytest.param( |
| 1373 | "glm46v", |
| 1374 | {"backend": "torchcodec"}, |
| 1375 | 120, |
| 1376 | id="glm46v-torchcodec-60s", |
| 1377 | ), |
| 1378 | ], |
| 1379 | ) |
| 1380 | def test_video_loader_frames_sampling( |
| 1381 | dummy_video_path, |
| 1382 | monkeypatch: pytest.MonkeyPatch, |
| 1383 | loader_key: str, |
| 1384 | kwargs: dict, |
| 1385 | expected_num_frames: int, |
| 1386 | ): |
| 1387 | """Test video loader frames sampling functionality.""" |
| 1388 | if kwargs.get("backend") == "torchcodec": |
| 1389 | pytest.importorskip("torchcodec") |
| 1390 | monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", loader_key) |
| 1391 | loader = VIDEO_LOADER_REGISTRY.load(loader_key) |
| 1392 | |
| 1393 | with open(dummy_video_path, "rb") as f: |
| 1394 | long_video_bytes = f.read() |
| 1395 | |
| 1396 | frames, _ = loader.load_bytes(long_video_bytes, **kwargs) |
| 1397 | |
| 1398 | assert frames.ndim == 4 |
| 1399 | assert frames.shape[3] == 3 # RGB |
| 1400 | assert frames.shape[0] == expected_num_frames |
| 1401 | |
| 1402 | |
| 1403 | # ============================================================================ |
| 1404 | # GLM-4.6V Dynamic FPS Threshold Tests |
| 1405 | # ============================================================================ |
| 1406 | |
| 1407 | |
| 1408 | @pytest.mark.parametrize( |
| 1409 | "duration, original_fps, total_frames, temporal_patch_size, expected_extract_t", |
| 1410 | [ |
| 1411 | # Short video ≤30s → target_fps=3.0 |
| 1412 | # extract_t = 10 * 3.0 * 2 = 60 |
| 1413 | pytest.param(10, 30, 300, 2, 60, id="short-10s"), |
| 1414 | # Exactly at boundary → target_fps=3.0 |
| 1415 | # extract_t = 30 * 3.0 * 2 = 180 |
| 1416 | pytest.param(30, 30, 900, 2, 180, id="boundary-30s"), |
| 1417 | # Medium video → target_fps=1.0 |
| 1418 | # extract_t = 60 * 1.0 * 2 = 120 |
| 1419 | pytest.param(60, 30, 1800, 2, 120, id="medium-60s"), |
| 1420 | # Medium boundary → target_fps=1.0 |
| 1421 | # extract_t = 300 * 1.0 * 2 = 600 |
| 1422 | pytest.param(300, 30, 9000, 2, 600, id="boundary-300s"), |
| 1423 | # Long video → target_fps=0.5 |
| 1424 | # extract_t = 600 * 0.5 * 2 = 600 |
| 1425 | pytest.param(600, 30, 18000, 2, 600, id="long-600s"), |
| 1426 | # Very long video, capped by _MAX_FRAME_COUNT_DYNAMIC=640 |
| 1427 | # extract_t = min(2400 * 0.5 * 2, 640) = min(2400, 640) = 640 |
| 1428 | pytest.param(2400, 30, 72000, 2, 640, id="long-capped-640"), |
| 1429 | # Duration exceeds _MAX_DURATION=2400 |
| 1430 | # effective_duration = min(5000, 2400) = 2400, target_fps=0.5 |
| 1431 | # extract_t = min(2400 * 0.5 * 2, 640) = 640 |
| 1432 | pytest.param(5000, 30, 150000, 2, 640, id="exceeds-max-duration"), |
| 1433 | # temporal_patch_size=4 |
| 1434 | # extract_t = 60 * 1.0 * 4 = 240 |
| 1435 | pytest.param(60, 30, 1800, 4, 240, id="medium-patch-size-4"), |
| 1436 | # temporal_patch_size=1 |
| 1437 | # extract_t = 60 * 1.0 * 1 = 60 |
| 1438 | pytest.param(60, 30, 1800, 1, 60, id="medium-patch-size-1"), |
| 1439 | ], |
| 1440 | ) |
| 1441 | def test_glm46v_dynamic_fps_thresholds( |
| 1442 | duration: int, |
| 1443 | original_fps: int, |
| 1444 | total_frames: int, |
| 1445 | temporal_patch_size: int, |
| 1446 | expected_extract_t: int, |
| 1447 | ): |
| 1448 | """Test GLM-4.6V dynamic FPS threshold selection and frame count.""" |
| 1449 | source = VideoSourceMetadata( |
| 1450 | total_frames_num=total_frames, |
| 1451 | original_fps=original_fps, |
| 1452 | duration=duration, |
| 1453 | ) |
| 1454 | target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1) |
| 1455 | |
| 1456 | indices = GLM46VVideoBackend.compute_frames_index_to_sample( |
| 1457 | source, target, temporal_patch_size=temporal_patch_size |
| 1458 | ) |
| 1459 | |
| 1460 | # Frame count should match expected (may be +1 from even padding) |
| 1461 | assert len(indices) in (expected_extract_t, expected_extract_t + 1), ( |
| 1462 | f"Expected ~{expected_extract_t} frames, got {len(indices)}" |
| 1463 | ) |
| 1464 | |
| 1465 | # Frame count must be even |
| 1466 | assert len(indices) % 2 == 0, f"Frame count must be even, got {len(indices)}" |
| 1467 | |
| 1468 | # All indices must be valid |
| 1469 | assert all(0 <= idx < total_frames for idx in indices), ( |
| 1470 | f"Indices out of range [0, {total_frames})" |
| 1471 | ) |
| 1472 | |
| 1473 | # Indices must be sorted and deduplicated |
| 1474 | assert indices == sorted(set(indices)), "Indices must be sorted and deduplicated" |
| 1475 | |
| 1476 | |
| 1477 | def test_glm46v_even_frame_count_enforcement(): |
| 1478 | """Test that GLM-4.6V always returns an even number of frames.""" |
| 1479 | target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1) |
| 1480 | # 5-second video at 30fps → 150 frames |
| 1481 | # extract_t = 5 * 3.0 * 2 = 30 (even, no padding needed) |
| 1482 | source_even = VideoSourceMetadata(total_frames_num=150, original_fps=30, duration=5) |
| 1483 | indices_even = GLM46VVideoBackend.compute_frames_index_to_sample( |
| 1484 | source_even, target |
| 1485 | ) |
| 1486 | assert len(indices_even) % 2 == 0 |
| 1487 | |
| 1488 | # 3-second video at 30fps → 90 frames |
| 1489 | # extract_t = 3 * 3.0 * 2 = 18 (even, no padding needed) |
| 1490 | source_even2 = VideoSourceMetadata(total_frames_num=90, original_fps=30, duration=3) |
| 1491 | indices_even2 = GLM46VVideoBackend.compute_frames_index_to_sample( |
| 1492 | source_even2, target |
| 1493 | ) |
| 1494 | assert len(indices_even2) % 2 == 0 |
| 1495 | |
| 1496 | |
| 1497 | def test_glm46v_duration_estimation_from_fps(): |
| 1498 | """Test GLM-4.6V handles missing duration by estimating from fps.""" |
| 1499 | target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1) |
| 1500 | # duration=0 → estimated from total_frames / fps |
| 1501 | # (89 / 30) + 1 ≈ 4s → target_fps=3.0, extract_t = 4 * 3.0 * 2 = 24 |
| 1502 | source_no_duration = VideoSourceMetadata( |
| 1503 | total_frames_num=90, original_fps=30, duration=0 |
| 1504 | ) |
| 1505 | indices = GLM46VVideoBackend.compute_frames_index_to_sample( |
| 1506 | source_no_duration, target |
| 1507 | ) |
| 1508 | |
| 1509 | assert len(indices) > 0 |
| 1510 | assert len(indices) % 2 == 0 |
| 1511 | assert all(0 <= idx < 90 for idx in indices) |
| 1512 | |
| 1513 | |
| 1514 | class TestGLMGASamplingCaps: |
| 1515 | """Regression tests for GHSA-58v5-2m8f-94pr: request-controlled fps |
| 1516 | and max_frames must not create intermediate allocations larger than |
| 1517 | the actual frame count.""" |
| 1518 | |
| 1519 | @staticmethod |
| 1520 | def _source( |
| 1521 | total_frames: int, fps: float = 30.0, duration: float = 0 |
| 1522 | ) -> VideoSourceMetadata: |
| 1523 | if duration == 0 and fps > 0 and total_frames > 1: |
| 1524 | duration = round((total_frames - 1) / fps) + 1 |
| 1525 | return VideoSourceMetadata(total_frames, fps, duration) |
| 1526 | |
| 1527 | def test_extreme_values_bounded_by_total_frames(self): |
| 1528 | source = self._source(total_frames=2, fps=2.0, duration=1.0) |
| 1529 | target = VideoTargetMetadata(num_frames=-1, fps=500_000, max_duration=-1) |
| 1530 | indices = GLMGAVideoBackend.compute_frames_index_to_sample( |
| 1531 | source, |
| 1532 | target, |
| 1533 | max_frames=500_000, |
| 1534 | ) |
| 1535 | assert len(indices) <= 2 |
| 1536 | |
| 1537 | def test_class_cap_overrides_kwargs_max_frames(self): |
| 1538 | source = self._source(total_frames=10_000, fps=30.0) |
| 1539 | target = VideoTargetMetadata(num_frames=-1, fps=30, max_duration=-1) |
| 1540 | indices = GLMGAVideoBackend.compute_frames_index_to_sample( |
| 1541 | source, |
| 1542 | target, |
| 1543 | max_frames=100_000, |
| 1544 | ) |
| 1545 | assert len(indices) <= GLMGAVideoBackend._MAX_FRAMES |
| 1546 | |
| 1547 | def test_class_cap_overrides_target_fps(self): |
| 1548 | source = self._source(total_frames=10_000, fps=30.0) |
| 1549 | target = VideoTargetMetadata(num_frames=-1, fps=500_000, max_duration=-1) |
| 1550 | indices = GLMGAVideoBackend.compute_frames_index_to_sample( |
| 1551 | source, |
| 1552 | target, |
| 1553 | ) |
| 1554 | assert len(indices) <= GLMGAVideoBackend._MAX_FRAMES |
| 1555 | |
| 1556 | def test_normal_operation_unchanged(self): |
| 1557 | source = self._source(total_frames=1000, fps=30.0, duration=33.0) |
| 1558 | target = VideoTargetMetadata(num_frames=-1, fps=2, max_duration=-1) |
| 1559 | indices = GLMGAVideoBackend.compute_frames_index_to_sample( |
| 1560 | source, |
| 1561 | target, |
| 1562 | ) |
| 1563 | assert 0 < len(indices) <= GLMGAVideoBackend._MAX_FRAMES |
| 1564 | assert all(0 <= idx < 1000 for idx in indices) |
| 1565 | |
| 1566 | |
| 1567 | def test_glm5next_backend_selected_for_processor(): |
| 1568 | """Glm5NextVideoProcessor maps to the glm5next loader so only the |
| 1569 | sampled frames are decoded instead of the whole container. Both the |
| 1570 | borrowed-config spelling and the dedicated Glm5next class name (landing |
| 1571 | with the new checkpoint) must resolve.""" |
| 1572 | for name in ("Glm5NextVideoProcessor", "Glm5nextVideoProcessor"): |
| 1573 | assert VIDEO_LOADER_REGISTRY.get_backend_for_video_processor(name) == "glm5next" |
| 1574 | |
| 1575 | |
| 1576 | @pytest.mark.parametrize( |
| 1577 | ("total_frames", "original_fps", "duration", "fps", "max_frames"), |
| 1578 | [ |
| 1579 | (900, 30.0, 30.0, -1, None), # 30s at flat 2.0 raw fps -> 60 frames |
| 1580 | (3000, 30.0, 100.0, -1, None), |
| 1581 | (72000, 30.0, 2400.0, -1, None), # 2048 cap |
| 1582 | (48, 2.0, 24.0, -1, None), |
| 1583 | (7, 30.0, 10.0, -1, None), # short video -> uniform spread + dedup |
| 1584 | (300, 25.0, 0, -1, None), # duration derived from frame count |
| 1585 | (900, 30.0, 30.0, 4, None), # request fps override (raw fps) |
| 1586 | (900, 30.0, 30.0, -1, 16), # request max_frames override |
| 1587 | ], |
| 1588 | ) |
| 1589 | def test_glm5next_backend_indices_match_sampler( |
| 1590 | total_frames, original_fps, duration, fps, max_frames |
| 1591 | ): |
| 1592 | """The loader must select exactly the frames the processor's sampler |
| 1593 | would, with target.fps mapping onto the raw-fps override.""" |
| 1594 | source = VideoSourceMetadata( |
| 1595 | total_frames_num=total_frames, original_fps=original_fps, duration=duration |
| 1596 | ) |
| 1597 | target = VideoTargetMetadata(num_frames=-1, fps=fps, max_duration=-1) |
| 1598 | |
| 1599 | indices = Glm5NextVideoBackend.compute_frames_index_to_sample( |
| 1600 | source, target, max_frames=max_frames |
| 1601 | ) |
| 1602 | |
| 1603 | assert indices == Glm5NextVideoBackend._sample_frame_indices( |
| 1604 | total_frames, |
| 1605 | original_fps, |
| 1606 | duration, |
| 1607 | target_fps=fps if fps > 0 else None, |
| 1608 | max_frame_count=max_frames, |
| 1609 | ) |
| 1610 | assert len(indices) % 2 == 0 |
| 1611 | assert indices == sorted(indices) # pair padding may repeat the last frame |
| 1612 | assert all(0 <= idx < total_frames for idx in indices) |
| 1613 | |
| 1614 | |
| 1615 | def test_glm5next_backend_metadata_contract(): |
| 1616 | """create_hf_metadata reports the subset so the processor skips |
| 1617 | re-sampling (do_sample_frames=False) and keeps the original totals.""" |
| 1618 | source = VideoSourceMetadata(total_frames_num=900, original_fps=30.0, duration=30.0) |
| 1619 | target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1) |
| 1620 | indices = Glm5NextVideoBackend.compute_frames_index_to_sample(source, target) |
| 1621 | |
| 1622 | metadata = Glm5NextVideoBackend.create_hf_metadata( |
| 1623 | source, indices, video_backend="glm5next" |
| 1624 | ) |
| 1625 | assert metadata["do_sample_frames"] is False |
| 1626 | assert metadata["frames_indices"] == indices |
| 1627 | assert metadata["total_num_frames"] == 900 |
| 1628 | assert metadata["fps"] == 30.0 |
| 1629 | assert metadata["duration"] == 30.0 |
| 1630 | |
| 1631 | # A fully-selected source keeps do_sample_frames=True so the processor's |
| 1632 | # sampler takes over on the complete frame set. |
| 1633 | full = list(range(48)) |
| 1634 | assert ( |
| 1635 | Glm5NextVideoBackend.create_hf_metadata( |
| 1636 | VideoSourceMetadata(total_frames_num=48, original_fps=2.0, duration=24.0), |
| 1637 | full, |
| 1638 | video_backend="glm5next", |
| 1639 | )["do_sample_frames"] |
| 1640 | is True |
| 1641 | ) |
| 1642 | |
| 1643 | |
| 1644 | def _write_gray_video(tmp_path, total_frames, fps, size=(32, 32)): |
| 1645 | """Synthetic clip whose frame i is flat gray level i (near-lossless under |
| 1646 | mp4v), so a decoded frame's level maps back to its source index.""" |
| 1647 | cv2 = pytest.importorskip("cv2") |
| 1648 | |
| 1649 | path = tmp_path / f"gray_{total_frames}_{fps}.mp4" |
| 1650 | writer = cv2.VideoWriter( |
| 1651 | str(path), cv2.VideoWriter_fourcc(*"mp4v"), fps, size, isColor=False |
| 1652 | ) |
| 1653 | assert writer.isOpened() |
| 1654 | for i in range(total_frames): |
| 1655 | writer.write(np.full((*size, 1), i, dtype=np.uint8)) |
| 1656 | writer.release() |
| 1657 | return path |
| 1658 | |
| 1659 | |
| 1660 | class _CountingCap: |
| 1661 | """Proxy over a real capture that counts grab/seek decoding work.""" |
| 1662 | |
| 1663 | def __init__(self, cap): |
| 1664 | self._cap = cap |
| 1665 | self.grabs = 0 |
| 1666 | self.seeks = 0 |
| 1667 | |
| 1668 | def get(self, prop): |
| 1669 | return self._cap.get(prop) |
| 1670 | |
| 1671 | def grab(self): |
| 1672 | self.grabs += 1 |
| 1673 | return self._cap.grab() |
| 1674 | |
| 1675 | def set(self, prop, value): |
| 1676 | self.seeks += 1 |
| 1677 | return self._cap.set(prop, value) |
| 1678 | |
| 1679 | def read(self): |
| 1680 | return self._cap.read() |
| 1681 | |
| 1682 | |
| 1683 | @pytest.mark.parametrize("backend", ["opencv", "torchcodec"]) |
| 1684 | def test_glm5next_backend_codec_parity(tmp_path, backend): |
| 1685 | """Every codec samples the same GLM indices and decodes the same |
| 1686 | frames; the OpenCV seek reader and torchcodec batched index-exact decode |
| 1687 | must agree.""" |
| 1688 | if backend == "torchcodec": |
| 1689 | pytest.importorskip("torchcodec") |
| 1690 | |
| 1691 | total_frames, fps = 120, 10 |
| 1692 | path = _write_gray_video(tmp_path, total_frames, fps) |
| 1693 | # Dense default sampling (gap 5) and a sparse max_frames cap (gap 20). |
| 1694 | for max_frames in (None, 6): |
| 1695 | kwargs: dict[str, Any] = ( |
| 1696 | {} if max_frames is None else {"max_frames": max_frames} |
| 1697 | ) |
| 1698 | expected = Glm5NextVideoBackend._sample_frame_indices( |
| 1699 | total_frames, float(fps), 12.0, max_frame_count=max_frames |
| 1700 | ) |
| 1701 | |
| 1702 | frames, metadata = Glm5NextVideoBackend.load_bytes( |
| 1703 | path.read_bytes(), backend=backend, **kwargs |
| 1704 | ) |
| 1705 | |
| 1706 | assert metadata["frames_indices"] == expected |
| 1707 | assert metadata["video_backend"].startswith(backend) |
| 1708 | assert len(frames) == len(expected) |
| 1709 | for i, idx in enumerate(expected): |
| 1710 | assert abs(round(float(np.asarray(frames[i]).mean())) - idx) <= 1 |
| 1711 | |
| 1712 | |
| 1713 | def test_glm5next_backend_decodes_only_sampled_frames(tmp_path): |
| 1714 | """End to end over a synthetic clip: load_bytes returns exactly the |
| 1715 | sampler's frame count, with the right frame content at each index.""" |
| 1716 | pytest.importorskip("cv2") |
| 1717 | |
| 1718 | total_frames, fps = 60, 10 |
| 1719 | path = _write_gray_video(tmp_path, total_frames, fps) |
| 1720 | |
| 1721 | expected = Glm5NextVideoBackend._sample_frame_indices(total_frames, float(fps), 6.0) |
| 1722 | |
| 1723 | frames, metadata = Glm5NextVideoBackend.load_bytes(path.read_bytes()) |
| 1724 | |
| 1725 | assert len(frames) == len(expected) |
| 1726 | assert metadata["frames_indices"] == expected |
| 1727 | assert metadata["do_sample_frames"] is (len(expected) == total_frames) |
| 1728 | # Flat gray frames survive mp4v near-losslessly: each decoded frame's |
| 1729 | # level maps back to its source frame index. |
| 1730 | for frame, idx in zip(frames, expected): |
| 1731 | decoded_idx = round(float(np.asarray(frame).mean())) |
| 1732 | assert abs(decoded_idx - idx) <= 1 |
| 1733 | |
| 1734 | |
| 1735 | def test_glm5next_read_frames_seeks_past_large_gaps(tmp_path): |
| 1736 | """Sparse targets must not walk the container: the stock reader grabs |
| 1737 | every frame up to the last index; the GLM reader seeks instead.""" |
| 1738 | cv2 = pytest.importorskip("cv2") |
| 1739 | |
| 1740 | total_frames, fps = 200, 10 |
| 1741 | path = _write_gray_video(tmp_path, total_frames, fps) |
| 1742 | targets = [0, 80, 160, 190] # gaps of 80/80/30 -> only 30 <= threshold |
| 1743 | |
| 1744 | stock = cv2.VideoCapture(str(path)) |
| 1745 | _, stock_indices = VideoBackend.read_frames(stock, targets, total_frames) |
| 1746 | stock.release() |
| 1747 | assert stock_indices == targets |
| 1748 | |
| 1749 | cap = _CountingCap(cv2.VideoCapture(str(path))) |
| 1750 | frames, indices = Glm5NextVideoBackend.read_frames(cap, targets, total_frames) |
| 1751 | cap._cap.release() |
| 1752 | |
| 1753 | assert indices == targets == stock_indices |
| 1754 | for frame, idx in zip(frames, targets): |
| 1755 | assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1 |
| 1756 | # Walks only the sub-threshold 30-frame hop; the two 80-frame gaps are |
| 1757 | # seeks. The stock reader grabs all 190 preceding frames. |
| 1758 | assert cap.grabs <= 29 |
| 1759 | assert cap.seeks == 3 |
| 1760 | |
| 1761 | |
| 1762 | def test_glm5next_read_frames_dense_walk_matches_stock(tmp_path): |
| 1763 | """Dense targets keep the sequential walk (seeking would be slower) and |
| 1764 | return the same frames as the stock reader.""" |
| 1765 | cv2 = pytest.importorskip("cv2") |
| 1766 | |
| 1767 | total_frames, fps = 120, 10 |
| 1768 | path = _write_gray_video(tmp_path, total_frames, fps) |
| 1769 | targets = list(range(0, total_frames, 15)) # gaps of 15 -> all walking |
| 1770 | |
| 1771 | stock = cv2.VideoCapture(str(path)) |
| 1772 | stock_frames, stock_indices = VideoBackend.read_frames(stock, targets, total_frames) |
| 1773 | stock.release() |
| 1774 | |
| 1775 | cap = _CountingCap(cv2.VideoCapture(str(path))) |
| 1776 | frames, indices = Glm5NextVideoBackend.read_frames(cap, targets, total_frames) |
| 1777 | cap._cap.release() |
| 1778 | |
| 1779 | assert indices == stock_indices |
| 1780 | for frame, stock_frame, idx in zip(frames, stock_frames, targets): |
| 1781 | assert abs(float(np.asarray(frame).mean()) - float(stock_frame.mean())) <= 2.0 |
| 1782 | assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1 |
| 1783 | # One initial seek, then pure walking -- no re-seek churn. |
| 1784 | assert cap.seeks == 1 |