| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import io |
| 4 | import sys |
| 5 | from pathlib import Path |
| 6 | from types import SimpleNamespace |
| 7 | |
| 8 | import numpy as np |
| 9 | import numpy.typing as npt |
| 10 | import pybase64 |
| 11 | import pytest |
| 12 | from PIL import Image |
| 13 | |
| 14 | from vllm.assets.base import get_vllm_public_assets |
| 15 | from vllm.assets.video import ( |
| 16 | video_get_metadata, |
| 17 | video_to_ndarrays, |
| 18 | video_to_pil_images_list, |
| 19 | ) |
| 20 | from vllm.multimodal.media import ImageMediaIO, VideoMediaIO |
| 21 | from vllm.multimodal.video import ( |
| 22 | PYNVVIDEOCODEC_VIDEO_BACKEND, |
| 23 | VIDEO_LOADER_REGISTRY, |
| 24 | VideoLoader, |
| 25 | ) |
| 26 | from vllm.multimodal.video_decoders.pynvvideocodec import ( |
| 27 | PyNvVideoCodecVideoBackendMixin, |
| 28 | _pynvvc_frames_to_nhwc, |
| 29 | ) |
| 30 | |
| 31 | from ..utils import cosine_similarity, create_video_from_image, normalize_image |
| 32 | |
| 33 | pytestmark = pytest.mark.cpu_test |
| 34 | |
| 35 | ASSETS_DIR = Path(__file__).parent.parent / "assets" |
| 36 | assert ASSETS_DIR.exists() |
| 37 | |
| 38 | |
| 39 | @VIDEO_LOADER_REGISTRY.register("assert_10_frames_1_fps") |
| 40 | class Assert10Frames1FPSVideoLoader(VideoLoader): |
| 41 | @classmethod |
| 42 | def load_bytes( |
| 43 | cls, data: bytes, num_frames: int = -1, fps: float = -1.0, **kwargs |
| 44 | ) -> npt.NDArray: |
| 45 | assert num_frames == 10, "bad num_frames" |
| 46 | assert fps == 1.0, "bad fps" |
| 47 | return FAKE_OUTPUT_2 |
| 48 | |
| 49 | |
| 50 | def test_video_media_io_kwargs(monkeypatch: pytest.MonkeyPatch): |
| 51 | with monkeypatch.context() as m: |
| 52 | m.setenv("VLLM_VIDEO_LOADER_BACKEND", "assert_10_frames_1_fps") |
| 53 | imageio = ImageMediaIO() |
| 54 | |
| 55 | # Verify that different args pass/fail assertions as expected. |
| 56 | videoio = VideoMediaIO(imageio, num_frames=10, fps=1.0) |
| 57 | _ = videoio.load_bytes(b"test") |
| 58 | |
| 59 | videoio = VideoMediaIO(imageio, num_frames=10, fps=1.0, not_used="not_used") |
| 60 | _ = videoio.load_bytes(b"test") |
| 61 | |
| 62 | with pytest.raises(AssertionError, match="bad num_frames"): |
| 63 | videoio = VideoMediaIO(imageio, **{}) |
| 64 | _ = videoio.load_bytes(b"test") |
| 65 | |
| 66 | with pytest.raises(AssertionError, match="bad num_frames"): |
| 67 | videoio = VideoMediaIO(imageio, num_frames=9, fps=1.0) |
| 68 | _ = videoio.load_bytes(b"test") |
| 69 | |
| 70 | with pytest.raises(AssertionError, match="bad fps"): |
| 71 | videoio = VideoMediaIO(imageio, num_frames=10, fps=2.0) |
| 72 | _ = videoio.load_bytes(b"test") |
| 73 | |
| 74 | |
| 75 | @pytest.mark.parametrize("is_color", [True, False]) |
| 76 | @pytest.mark.parametrize("fourcc, ext", [("mp4v", "mp4"), ("XVID", "avi")]) |
| 77 | def test_opencv_video_io_colorspace(tmp_path, is_color: bool, fourcc: str, ext: str): |
| 78 | """Test all functions that use OpenCV for video I/O return RGB format. |
| 79 | Both RGB and grayscale videos are tested. |
| 80 | """ |
| 81 | image_path = get_vllm_public_assets( |
| 82 | filename="stop_sign.jpg", s3_prefix="vision_model_images" |
| 83 | ) |
| 84 | image = Image.open(image_path) |
| 85 | |
| 86 | if not is_color: |
| 87 | image_path = Path(tmp_path) / "test_grayscale_image.png" |
| 88 | image = image.convert("L") |
| 89 | image.save(image_path) |
| 90 | # Convert to gray RGB for comparison |
| 91 | image = image.convert("RGB") |
| 92 | video_path = f"{tmp_path}/test_RGB_video.{ext}" |
| 93 | create_video_from_image( |
| 94 | str(image_path), |
| 95 | video_path, |
| 96 | num_frames=2, |
| 97 | is_color=is_color, |
| 98 | fourcc=fourcc, |
| 99 | ) |
| 100 | |
| 101 | frames = video_to_ndarrays(video_path) |
| 102 | for frame in frames: |
| 103 | sim = cosine_similarity( |
| 104 | normalize_image(np.array(frame)), normalize_image(np.array(image)) |
| 105 | ) |
| 106 | assert np.sum(np.isnan(sim)) / sim.size < 0.001 |
| 107 | assert np.nanmean(sim) > 0.99 |
| 108 | |
| 109 | pil_frames = video_to_pil_images_list(video_path) |
| 110 | for frame in pil_frames: |
| 111 | sim = cosine_similarity( |
| 112 | normalize_image(np.array(frame)), normalize_image(np.array(image)) |
| 113 | ) |
| 114 | assert np.sum(np.isnan(sim)) / sim.size < 0.001 |
| 115 | assert np.nanmean(sim) > 0.99 |
| 116 | |
| 117 | io_frames, _ = VideoMediaIO(ImageMediaIO()).load_file(Path(video_path)) |
| 118 | for frame in io_frames: |
| 119 | sim = cosine_similarity( |
| 120 | normalize_image(np.array(frame)), normalize_image(np.array(image)) |
| 121 | ) |
| 122 | assert np.sum(np.isnan(sim)) / sim.size < 0.001 |
| 123 | assert np.nanmean(sim) > 0.99 |
| 124 | |
| 125 | |
| 126 | def test_opencv_video_metadata_matches_sampled_frame_timeline(tmp_path): |
| 127 | image_path = f"{tmp_path}/test_metadata_image.png" |
| 128 | Image.new("RGB", (8, 8), color=(255, 0, 0)).save(image_path) |
| 129 | video_path = f"{tmp_path}/test_metadata_video.mp4" |
| 130 | create_video_from_image(image_path, video_path, num_frames=10, fps=5.0) |
| 131 | |
| 132 | metadata = video_get_metadata(video_path, num_frames=4) |
| 133 | |
| 134 | assert metadata["fps"] == pytest.approx(5.0) |
| 135 | assert metadata["duration"] == pytest.approx(2.0) |
| 136 | assert metadata["frames_indices"] == [0, 3, 6, 9] |
| 137 | assert metadata["total_num_frames"] == 4 |
| 138 | |
| 139 | |
| 140 | NUM_FRAMES = 10 |
| 141 | FAKE_OUTPUT_1 = np.random.rand(NUM_FRAMES, 1280, 720, 3) |
| 142 | FAKE_OUTPUT_2 = np.random.rand(NUM_FRAMES, 1280, 720, 3) |
| 143 | |
| 144 | |
| 145 | @VIDEO_LOADER_REGISTRY.register("test_video_backend_override_1") |
| 146 | class TestVideoBackendOverride1(VideoLoader): |
| 147 | """Test loader that returns FAKE_OUTPUT_1 to verify backend selection.""" |
| 148 | |
| 149 | @classmethod |
| 150 | def load_bytes( |
| 151 | cls, data: bytes, num_frames: int = -1, **kwargs |
| 152 | ) -> tuple[npt.NDArray, dict]: |
| 153 | return FAKE_OUTPUT_1, {"video_backend": "test_video_backend_override_1"} |
| 154 | |
| 155 | |
| 156 | @VIDEO_LOADER_REGISTRY.register("test_video_backend_override_2") |
| 157 | class TestVideoBackendOverride2(VideoLoader): |
| 158 | """Test loader that returns FAKE_OUTPUT_2 to verify backend selection.""" |
| 159 | |
| 160 | @classmethod |
| 161 | def load_bytes( |
| 162 | cls, data: bytes, num_frames: int = -1, **kwargs |
| 163 | ) -> tuple[npt.NDArray, dict]: |
| 164 | return FAKE_OUTPUT_2, {"video_backend": "test_video_backend_override_2"} |
| 165 | |
| 166 | |
| 167 | def test_video_media_io_backend_kwarg_override(monkeypatch: pytest.MonkeyPatch): |
| 168 | """Test that video_backend kwarg can override the VLLM_VIDEO_LOADER_BACKEND |
| 169 | environment variable. |
| 170 | |
| 171 | This allows users to dynamically select a different video backend |
| 172 | via --media-io-kwargs without changing the global env var, which is |
| 173 | useful when plugins set a default backend but a specific request |
| 174 | needs a different one. |
| 175 | """ |
| 176 | with monkeypatch.context() as m: |
| 177 | # Set the env var to one backend |
| 178 | m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_video_backend_override_1") |
| 179 | |
| 180 | imageio = ImageMediaIO() |
| 181 | |
| 182 | # Without video_backend kwarg, should use env var backend |
| 183 | videoio_default = VideoMediaIO(imageio, num_frames=10) |
| 184 | frames_default, metadata_default = videoio_default.load_bytes(b"test") |
| 185 | np.testing.assert_array_equal(frames_default, FAKE_OUTPUT_1) |
| 186 | assert metadata_default["video_backend"] == "test_video_backend_override_1" |
| 187 | |
| 188 | # With video_backend kwarg, should override env var |
| 189 | videoio_override = VideoMediaIO( |
| 190 | imageio, num_frames=10, video_backend="test_video_backend_override_2" |
| 191 | ) |
| 192 | frames_override, metadata_override = videoio_override.load_bytes(b"test") |
| 193 | np.testing.assert_array_equal(frames_override, FAKE_OUTPUT_2) |
| 194 | assert metadata_override["video_backend"] == "test_video_backend_override_2" |
| 195 | |
| 196 | |
| 197 | def test_video_media_io_backend_kwarg_not_passed_to_loader( |
| 198 | monkeypatch: pytest.MonkeyPatch, |
| 199 | ): |
| 200 | """Test that video_backend kwarg is consumed by VideoMediaIO and NOT passed |
| 201 | through to the underlying video loader's load_bytes method. |
| 202 | |
| 203 | This ensures the kwarg is properly popped from kwargs before forwarding. |
| 204 | """ |
| 205 | |
| 206 | @VIDEO_LOADER_REGISTRY.register("test_reject_video_backend_kwarg") |
| 207 | class RejectVideoBackendKwargLoader(VideoLoader): |
| 208 | """Test loader that fails if video_backend is passed through.""" |
| 209 | |
| 210 | @classmethod |
| 211 | def load_bytes( |
| 212 | cls, data: bytes, num_frames: int = -1, **kwargs |
| 213 | ) -> tuple[npt.NDArray, dict]: |
| 214 | # This should never receive video_backend in kwargs |
| 215 | if "video_backend" in kwargs: |
| 216 | raise AssertionError( |
| 217 | "video_backend should be consumed by VideoMediaIO, " |
| 218 | "not passed to loader" |
| 219 | ) |
| 220 | return FAKE_OUTPUT_1, {"received_kwargs": list(kwargs.keys())} |
| 221 | |
| 222 | with monkeypatch.context() as m: |
| 223 | m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_reject_video_backend_kwarg") |
| 224 | |
| 225 | imageio = ImageMediaIO() |
| 226 | |
| 227 | # Even when video_backend is provided, it should NOT be passed to loader |
| 228 | videoio = VideoMediaIO( |
| 229 | imageio, |
| 230 | num_frames=10, |
| 231 | video_backend="test_reject_video_backend_kwarg", |
| 232 | other_kwarg="should_pass_through", |
| 233 | ) |
| 234 | |
| 235 | # This should NOT raise AssertionError |
| 236 | frames, metadata = videoio.load_bytes(b"test") |
| 237 | np.testing.assert_array_equal(frames, FAKE_OUTPUT_1) |
| 238 | # Verify other kwargs are still passed through |
| 239 | assert "other_kwarg" in metadata["received_kwargs"] |
| 240 | |
| 241 | |
| 242 | def test_video_media_io_backend_env_var_fallback(monkeypatch: pytest.MonkeyPatch): |
| 243 | """Test that when video_backend kwarg is None or not provided, |
| 244 | VideoMediaIO falls back to VLLM_VIDEO_LOADER_BACKEND env var. |
| 245 | """ |
| 246 | with monkeypatch.context() as m: |
| 247 | m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_video_backend_override_2") |
| 248 | |
| 249 | imageio = ImageMediaIO() |
| 250 | |
| 251 | # Explicit None should fall back to env var |
| 252 | videoio_none = VideoMediaIO(imageio, num_frames=10, video_backend=None) |
| 253 | frames_none, metadata_none = videoio_none.load_bytes(b"test") |
| 254 | np.testing.assert_array_equal(frames_none, FAKE_OUTPUT_2) |
| 255 | assert metadata_none["video_backend"] == "test_video_backend_override_2" |
| 256 | |
| 257 | # Not providing video_backend should also fall back to env var |
| 258 | videoio_missing = VideoMediaIO(imageio, num_frames=10) |
| 259 | frames_missing, metadata_missing = videoio_missing.load_bytes(b"test") |
| 260 | np.testing.assert_array_equal(frames_missing, FAKE_OUTPUT_2) |
| 261 | assert metadata_missing["video_backend"] == "test_video_backend_override_2" |
| 262 | |
| 263 | |
| 264 | def _make_jpeg_b64_frames(n: int, width: int = 8, height: int = 8) -> list[str]: |
| 265 | """Return *n* tiny base64-encoded JPEG frames.""" |
| 266 | frames: list[str] = [] |
| 267 | for i in range(n): |
| 268 | img = Image.new("RGB", (width, height), color=(i % 256, 0, 0)) |
| 269 | buf = io.BytesIO() |
| 270 | img.save(buf, format="JPEG") |
| 271 | frames.append(pybase64.b64encode(buf.getvalue()).decode("ascii")) |
| 272 | return frames |
| 273 | |
| 274 | |
| 275 | def test_load_base64_jpeg_returns_metadata(): |
| 276 | """Regression test: load_base64 with video/jpeg must return metadata. |
| 277 | |
| 278 | Previously, base64 JPEG frame sequences returned an empty dict for |
| 279 | metadata, which broke downstream consumers that rely on fields like |
| 280 | total_num_frames and fps. See PR #37301. |
| 281 | """ |
| 282 | num_test_frames = 3 |
| 283 | |
| 284 | b64_frames = _make_jpeg_b64_frames(num_test_frames) |
| 285 | data = ",".join(b64_frames) |
| 286 | |
| 287 | imageio = ImageMediaIO() |
| 288 | videoio = VideoMediaIO(imageio, num_frames=num_test_frames) |
| 289 | frames, metadata = videoio.load_base64("video/jpeg", data) |
| 290 | |
| 291 | # Frames array shape: (num_frames, H, W, 3) |
| 292 | assert frames.shape[0] == num_test_frames |
| 293 | |
| 294 | # All required metadata keys must be present |
| 295 | required_keys = { |
| 296 | "total_num_frames", |
| 297 | "fps", |
| 298 | "duration", |
| 299 | "video_backend", |
| 300 | "frames_indices", |
| 301 | "do_sample_frames", |
| 302 | } |
| 303 | assert required_keys.issubset(metadata.keys()), ( |
| 304 | f"Missing metadata keys: {required_keys - metadata.keys()}" |
| 305 | ) |
| 306 | |
| 307 | assert metadata["total_num_frames"] == num_test_frames |
| 308 | assert metadata["video_backend"] == "jpeg_sequence" |
| 309 | assert metadata["frames_indices"] == list(range(num_test_frames)) |
| 310 | assert metadata["do_sample_frames"] is False |
| 311 | # Default fps=1 → duration == num_frames |
| 312 | assert metadata["fps"] == 1.0 |
| 313 | assert metadata["duration"] == float(num_test_frames) |
| 314 | |
| 315 | |
| 316 | def test_load_base64_jpeg_enforces_num_frames_limit(): |
| 317 | """Frames beyond num_frames must be truncated in the video/jpeg path. |
| 318 | |
| 319 | Without the limit an attacker can send thousands of base64 JPEG frames |
| 320 | in a single request and exhaust server memory (OOM). |
| 321 | """ |
| 322 | num_frames_limit = 4 |
| 323 | sent_frames = 20 |
| 324 | |
| 325 | b64_frames = _make_jpeg_b64_frames(sent_frames) |
| 326 | data = ",".join(b64_frames) |
| 327 | |
| 328 | imageio = ImageMediaIO() |
| 329 | videoio = VideoMediaIO(imageio, num_frames=num_frames_limit) |
| 330 | frames, metadata = videoio.load_base64("video/jpeg", data) |
| 331 | |
| 332 | assert frames.shape[0] == num_frames_limit |
| 333 | assert metadata["total_num_frames"] == num_frames_limit |
| 334 | assert metadata["frames_indices"] == list(range(num_frames_limit)) |
| 335 | |
| 336 | |
| 337 | def test_load_base64_jpeg_no_limit_when_num_frames_negative(): |
| 338 | """When num_frames is -1, all frames should be loaded without truncation.""" |
| 339 | sent_frames = 10 |
| 340 | |
| 341 | b64_frames = _make_jpeg_b64_frames(sent_frames) |
| 342 | data = ",".join(b64_frames) |
| 343 | |
| 344 | imageio = ImageMediaIO() |
| 345 | videoio = VideoMediaIO(imageio, num_frames=-1) |
| 346 | frames, metadata = videoio.load_base64("video/jpeg", data) |
| 347 | |
| 348 | assert frames.shape[0] == sent_frames |
| 349 | assert metadata["total_num_frames"] == sent_frames |
| 350 | assert metadata["frames_indices"] == list(range(sent_frames)) |
| 351 | |
| 352 | |
| 353 | def test_load_base64_jpeg_raises_on_zero_num_frames(): |
| 354 | """num_frames=0 is invalid and should raise ValueError.""" |
| 355 | b64_frames = _make_jpeg_b64_frames(3) |
| 356 | data = ",".join(b64_frames) |
| 357 | |
| 358 | imageio = ImageMediaIO() |
| 359 | videoio = VideoMediaIO(imageio, num_frames=0) |
| 360 | |
| 361 | with pytest.raises(ValueError, match="num_frames must be greater than 0 or -1"): |
| 362 | videoio.load_base64("video/jpeg", data) |
| 363 | |
| 364 | |
| 365 | def test_pynvvideocodec_unrelated_error_propagates( |
| 366 | monkeypatch: pytest.MonkeyPatch, |
| 367 | ): |
| 368 | class FakePyNvVCException(Exception): |
| 369 | pass |
| 370 | |
| 371 | fake_nvc = SimpleNamespace(PyNvVCException=FakePyNvVCException) |
| 372 | monkeypatch.setitem(sys.modules, "PyNvVideoCodec", fake_nvc) |
| 373 | original_error = RuntimeError("GPU decoder unavailable") |
| 374 | |
| 375 | def raise_unrelated_error(cls, file_path, nvc): |
| 376 | raise original_error |
| 377 | |
| 378 | monkeypatch.setattr( |
| 379 | PyNvVideoCodecVideoBackendMixin, |
| 380 | "_read_source_metadata", |
| 381 | classmethod(raise_unrelated_error), |
| 382 | ) |
| 383 | |
| 384 | with pytest.raises(RuntimeError) as exc_info: |
| 385 | PyNvVideoCodecVideoBackendMixin.decode_frames_pynvvideocodec( |
| 386 | None, |
| 387 | b"video", |
| 388 | None, |
| 389 | ) |
| 390 | |
| 391 | assert exc_info.value is original_error |
| 392 | |
| 393 | |
| 394 | # --------------------------------------------------------------------------- |
| 395 | # GPU video backend policy tests |
| 396 | # --------------------------------------------------------------------------- |
| 397 | |
| 398 | |
| 399 | class TestMergeKwargsGpuBackendPolicy: |
| 400 | """Verify that merge_kwargs blocks request-level GPU backend selection |
| 401 | when the static (engine-level) config did not configure that backend.""" |
| 402 | |
| 403 | def test_pynvvideocodec_requires_gpu(self): |
| 404 | assert VIDEO_LOADER_REGISTRY.backend_requires_gpu(PYNVVIDEOCODEC_VIDEO_BACKEND) |
| 405 | |
| 406 | def test_strips_video_backend_pynv_when_not_static(self): |
| 407 | result = VideoMediaIO.merge_kwargs( |
| 408 | default_kwargs=None, |
| 409 | runtime_kwargs={"video_backend": "pynvvideocodec"}, |
| 410 | ) |
| 411 | assert "video_backend" not in result |
| 412 | |
| 413 | def test_strips_backend_pynv_when_not_static(self): |
| 414 | result = VideoMediaIO.merge_kwargs( |
| 415 | default_kwargs={"num_frames": 16}, |
| 416 | runtime_kwargs={"backend": "pynvvideocodec"}, |
| 417 | ) |
| 418 | assert result.get("backend") != "pynvvideocodec" |
| 419 | |
| 420 | def test_preserves_video_backend_pynv_when_static(self): |
| 421 | result = VideoMediaIO.merge_kwargs( |
| 422 | default_kwargs={"video_backend": "pynvvideocodec"}, |
| 423 | runtime_kwargs={"video_backend": "pynvvideocodec", "num_frames": 8}, |
| 424 | ) |
| 425 | assert result["video_backend"] == "pynvvideocodec" |
| 426 | assert result["num_frames"] == 8 |
| 427 | |
| 428 | def test_preserves_backend_pynv_when_static(self): |
| 429 | result = VideoMediaIO.merge_kwargs( |
| 430 | default_kwargs={"backend": "pynvvideocodec"}, |
| 431 | runtime_kwargs={"backend": "pynvvideocodec"}, |
| 432 | ) |
| 433 | assert result["backend"] == "pynvvideocodec" |
| 434 | |
| 435 | def test_strips_request_level_hw_decoders_when_not_static(self): |
| 436 | result = VideoMediaIO.merge_kwargs( |
| 437 | default_kwargs={"video_backend": "pynvvideocodec"}, |
| 438 | runtime_kwargs={"hw_decoders": 4}, |
| 439 | ) |
| 440 | assert "hw_decoders" not in result |
| 441 | |
| 442 | def test_prevents_request_level_hw_decoders_override(self): |
| 443 | result = VideoMediaIO.merge_kwargs( |
| 444 | default_kwargs={ |
| 445 | "video_backend": "pynvvideocodec", |
| 446 | "hw_decoders": 2, |
| 447 | }, |
| 448 | runtime_kwargs={"hw_decoders": 4}, |
| 449 | ) |
| 450 | assert result["hw_decoders"] == 2 |
| 451 | |
| 452 | @pytest.mark.parametrize("backend", ["opencv", "torchcodec"]) |
| 453 | def test_software_video_backend_passes_through(self, backend: str): |
| 454 | result = VideoMediaIO.merge_kwargs( |
| 455 | default_kwargs=None, |
| 456 | runtime_kwargs={"video_backend": backend}, |
| 457 | ) |
| 458 | assert result["video_backend"] == backend |
| 459 | |
| 460 | @pytest.mark.parametrize("backend", ["opencv"]) |
| 461 | def test_software_codec_backend_passes_through(self, backend: str): |
| 462 | result = VideoMediaIO.merge_kwargs( |
| 463 | default_kwargs=None, |
| 464 | runtime_kwargs={"backend": backend}, |
| 465 | ) |
| 466 | assert result["backend"] == backend |
| 467 | |
| 468 | def test_strips_both_keys_independently(self): |
| 469 | result = VideoMediaIO.merge_kwargs( |
| 470 | default_kwargs=None, |
| 471 | runtime_kwargs={ |
| 472 | "video_backend": "pynvvideocodec", |
| 473 | "backend": "pynvvideocodec", |
| 474 | "num_frames": 4, |
| 475 | }, |
| 476 | ) |
| 477 | assert "video_backend" not in result |
| 478 | assert result.get("backend") != "pynvvideocodec" |
| 479 | assert result["num_frames"] == 4 |
| 480 | |
| 481 | def test_other_kwargs_preserved_when_gpu_backend_stripped(self): |
| 482 | result = VideoMediaIO.merge_kwargs( |
| 483 | default_kwargs={"fps": 2}, |
| 484 | runtime_kwargs={ |
| 485 | "video_backend": "pynvvideocodec", |
| 486 | "num_frames": 16, |
| 487 | }, |
| 488 | ) |
| 489 | assert "video_backend" not in result |
| 490 | assert result["num_frames"] == 16 |
| 491 | |
| 492 | def test_static_pynv_with_different_runtime_gpu_backend(self): |
| 493 | """If static sets pynv via video_backend but runtime tries to set it |
| 494 | via the codec-level 'backend' key (without a static match), strip it.""" |
| 495 | result = VideoMediaIO.merge_kwargs( |
| 496 | default_kwargs={"video_backend": "pynvvideocodec"}, |
| 497 | runtime_kwargs={"backend": "pynvvideocodec"}, |
| 498 | ) |
| 499 | assert result.get("backend") != "pynvvideocodec" |
| 500 | assert result["video_backend"] == "pynvvideocodec" |
| 501 | |
| 502 | def test_deepstream_requires_gpu(self): |
| 503 | assert VIDEO_LOADER_REGISTRY.backend_requires_gpu("deepstream") |
| 504 | |
| 505 | def test_strips_backend_deepstream_when_not_static(self): |
| 506 | result = VideoMediaIO.merge_kwargs( |
| 507 | default_kwargs=None, |
| 508 | runtime_kwargs={"backend": "deepstream"}, |
| 509 | ) |
| 510 | assert result.get("backend") != "deepstream" |
| 511 | |
| 512 | def test_preserves_backend_deepstream_when_static(self): |
| 513 | result = VideoMediaIO.merge_kwargs( |
| 514 | default_kwargs={"backend": "deepstream"}, |
| 515 | runtime_kwargs={"backend": "deepstream", "num_frames": 8}, |
| 516 | ) |
| 517 | assert result["backend"] == "deepstream" |
| 518 | assert result["num_frames"] == 8 |
| 519 | |
| 520 | def test_strips_pool_size_from_runtime(self): |
| 521 | result = VideoMediaIO.merge_kwargs( |
| 522 | default_kwargs={"backend": "deepstream"}, |
| 523 | runtime_kwargs={"backend": "deepstream", "pool_size": 4}, |
| 524 | ) |
| 525 | assert "pool_size" not in result |
| 526 | |
| 527 | def test_unknown_backend_not_treated_as_gpu(self): |
| 528 | assert not VIDEO_LOADER_REGISTRY.backend_requires_gpu("totally_unknown") |
| 529 | |
| 530 | def test_strips_request_level_device(self): |
| 531 | """The decode device is a startup-only knob: a request must not move |
| 532 | decoding onto the GPU when the startup config did not opt in, nor off |
| 533 | it when it did.""" |
| 534 | result = VideoMediaIO.merge_kwargs( |
| 535 | default_kwargs={"backend": "torchcodec"}, |
| 536 | runtime_kwargs={"device": "cuda"}, |
| 537 | ) |
| 538 | assert "device" not in result |
| 539 | |
| 540 | result = VideoMediaIO.merge_kwargs( |
| 541 | default_kwargs={"backend": "torchcodec", "device": "cuda"}, |
| 542 | runtime_kwargs={"device": "cpu", "num_frames": 8}, |
| 543 | ) |
| 544 | assert result["device"] == "cuda" |
| 545 | assert result["num_frames"] == 8 |
| 546 | |
| 547 | |
| 548 | @pytest.mark.parametrize( |
| 549 | "default_kwargs", |
| 550 | [ |
| 551 | {"backend": "torchcodec", "device": "cuda", "seek_mode": "approximate"}, |
| 552 | {"backend": "pynvvideocodec", "hw_decoders": 2}, |
| 553 | ], |
| 554 | ) |
| 555 | def test_switching_backend_drops_stale_codec_options(default_kwargs): |
| 556 | """Codec-specific options from the static config must not leak into a |
| 557 | different codec backend selected per-request, where they would fail the |
| 558 | new backend's option validation.""" |
| 559 | result = VideoMediaIO.merge_kwargs( |
| 560 | default_kwargs={**default_kwargs, "num_frames": 8}, |
| 561 | runtime_kwargs={"backend": "opencv"}, |
| 562 | ) |
| 563 | assert result == {"backend": "opencv", "num_frames": 8} |
| 564 | |
| 565 | |
| 566 | def test_same_backend_keeps_codec_options(): |
| 567 | result = VideoMediaIO.merge_kwargs( |
| 568 | default_kwargs={"backend": "torchcodec", "device": "cuda"}, |
| 569 | runtime_kwargs={"backend": "torchcodec", "num_frames": 8}, |
| 570 | ) |
| 571 | assert result["device"] == "cuda" |
| 572 | assert result["num_frames"] == 8 |
| 573 | |
| 574 | |
| 575 | @pytest.mark.parametrize("layout", ["nhwc", "nchw"]) |
| 576 | def test_pynvvc_frames_normalized_to_nhwc(layout: str): |
| 577 | """PyNvVideoCodec frame batches are normalized to NHWC regardless of the |
| 578 | per-frame layout the decoder emits (it has varied across versions), so the |
| 579 | HF video processors (which materialize a PIL image per frame) receive the |
| 580 | same NHWC shape as every other video backend.""" |
| 581 | torch = pytest.importorskip("torch") |
| 582 | |
| 583 | n, h, w, c = 4, 5, 6, 3 |
| 584 | nhwc = torch.arange(n * h * w * c, dtype=torch.uint8).reshape(n, h, w, c) |
| 585 | frames = nhwc if layout == "nhwc" else nhwc.permute(0, 3, 1, 2).contiguous() |
| 586 | |
| 587 | out = _pynvvc_frames_to_nhwc(frames) |
| 588 | |
| 589 | assert out.shape == (n, h, w, c) |
| 590 | assert out.is_contiguous() |
| 591 | assert torch.equal(out, nhwc) # content preserved / correctly transposed |