fix(security): block request-level GPU video backend selection withou… (#47259)
Signed-off-by: jperezde <jperezde@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Juan Pérez de Algaba committed
Jul 6, 2026 at 11:36 UTC
ba22152096b2484faa3579624a253d54804d876d
4 files changed
+131
-8
tests/multimodal/media/test_video.py
+97
-1
@@ -16,7 +16,11 @@ from vllm.assets.video import (
16
video_to_pil_images_list,
17
)
18
from vllm.multimodal.media import ImageMediaIO, VideoMediaIO
19
-from vllm.multimodal.video import VIDEO_LOADER_REGISTRY, VideoLoader
19
+from vllm.multimodal.video import (
20
+ PYNVVIDEOCODEC_VIDEO_BACKEND,
21
+ VIDEO_LOADER_REGISTRY,
22
+ VideoLoader,
23
+)
24
25
from ..utils import cosine_similarity, create_video_from_image, normalize_image
26
@@ -357,3 +361,95 @@ def test_load_base64_jpeg_raises_on_zero_num_frames():
361
362
with pytest.raises(ValueError, match="num_frames must be greater than 0 or -1"):
363
videoio.load_base64("video/jpeg", data)
364
+
365
+
366
+# ---------------------------------------------------------------------------
367
+# GPU video backend policy tests
368
+# ---------------------------------------------------------------------------
369
+
370
+
371
+class TestMergeKwargsGpuBackendPolicy:
372
+ """Verify that merge_kwargs blocks request-level GPU backend selection
373
+ when the static (engine-level) config did not configure that backend."""
374
+
375
+ def test_pynvvideocodec_requires_gpu(self):
376
+ assert VIDEO_LOADER_REGISTRY.backend_requires_gpu(PYNVVIDEOCODEC_VIDEO_BACKEND)
377
+
378
+ def test_strips_video_backend_pynv_when_not_static(self):
379
+ result = VideoMediaIO.merge_kwargs(
380
+ default_kwargs=None,
381
+ runtime_kwargs={"video_backend": "pynvvideocodec"},
382
+ )
383
+ assert "video_backend" not in result
384
+
385
+ def test_strips_backend_pynv_when_not_static(self):
386
+ result = VideoMediaIO.merge_kwargs(
387
+ default_kwargs={"num_frames": 16},
388
+ runtime_kwargs={"backend": "pynvvideocodec"},
389
+ )
390
+ assert result.get("backend") != "pynvvideocodec"
391
+
392
+ def test_preserves_video_backend_pynv_when_static(self):
393
+ result = VideoMediaIO.merge_kwargs(
394
+ default_kwargs={"video_backend": "pynvvideocodec"},
395
+ runtime_kwargs={"video_backend": "pynvvideocodec", "num_frames": 8},
396
+ )
397
+ assert result["video_backend"] == "pynvvideocodec"
398
+ assert result["num_frames"] == 8
399
+
400
+ def test_preserves_backend_pynv_when_static(self):
401
+ result = VideoMediaIO.merge_kwargs(
402
+ default_kwargs={"backend": "pynvvideocodec"},
403
+ runtime_kwargs={"backend": "pynvvideocodec"},
404
+ )
405
+ assert result["backend"] == "pynvvideocodec"
406
+
407
+ @pytest.mark.parametrize("backend", ["opencv", "pyav", "torchcodec"])
408
+ def test_software_video_backend_passes_through(self, backend: str):
409
+ result = VideoMediaIO.merge_kwargs(
410
+ default_kwargs=None,
411
+ runtime_kwargs={"video_backend": backend},
412
+ )
413
+ assert result["video_backend"] == backend
414
+
415
+ @pytest.mark.parametrize("backend", ["opencv", "pyav"])
416
+ def test_software_codec_backend_passes_through(self, backend: str):
417
+ result = VideoMediaIO.merge_kwargs(
418
+ default_kwargs=None,
419
+ runtime_kwargs={"backend": backend},
420
+ )
421
+ assert result["backend"] == backend
422
+
423
+ def test_strips_both_keys_independently(self):
424
+ result = VideoMediaIO.merge_kwargs(
425
+ default_kwargs=None,
426
+ runtime_kwargs={
427
+ "video_backend": "pynvvideocodec",
428
+ "backend": "pynvvideocodec",
429
+ "num_frames": 4,
430
+ },
431
+ )
432
+ assert "video_backend" not in result
433
+ assert result.get("backend") != "pynvvideocodec"
434
+ assert result["num_frames"] == 4
435
+
436
+ def test_other_kwargs_preserved_when_gpu_backend_stripped(self):
437
+ result = VideoMediaIO.merge_kwargs(
438
+ default_kwargs={"fps": 2},
439
+ runtime_kwargs={
440
+ "video_backend": "pynvvideocodec",
441
+ "num_frames": 16,
442
+ },
443
+ )
444
+ assert "video_backend" not in result
445
+ assert result["num_frames"] == 16
446
+
447
+ def test_static_pynv_with_different_runtime_gpu_backend(self):
448
+ """If static sets pynv via video_backend but runtime tries to set it
449
+ via the codec-level 'backend' key (without a static match), strip it."""
450
+ result = VideoMediaIO.merge_kwargs(
451
+ default_kwargs={"video_backend": "pynvvideocodec"},
452
+ runtime_kwargs={"backend": "pynvvideocodec"},
453
+ )
454
+ assert result.get("backend") != "pynvvideocodec"
455
+ assert result["video_backend"] == "pynvvideocodec"
vllm/multimodal/media/video.py
+21
@@ -10,11 +10,14 @@ import pybase64
10
from PIL import Image
11
12
from vllm import envs
13
+from vllm.logger import init_logger
14
15
from ..video import VIDEO_LOADER_REGISTRY
16
from .base import MediaIO
17
from .image import ImageMediaIO
18
19
+logger = init_logger(__name__)
20
+
21
22
class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
23
"""Configuration values can be user-provided either by --media-io-kwargs or
@@ -28,6 +31,24 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
31
default_kwargs: dict[str, Any] | None,
32
runtime_kwargs: dict[str, Any] | None,
33
) -> dict[str, Any]:
34
+ if runtime_kwargs:
35
+ # Block request-level selection of GPU video backends that
36
+ # were not configured (and VRAM-reserved) at startup.
37
+ for key in ("video_backend", "backend"):
38
+ requested = runtime_kwargs.get(key)
39
+ if requested and VIDEO_LOADER_REGISTRY.backend_requires_gpu(requested):
40
+ static_val = (default_kwargs or {}).get(key)
41
+ if static_val != requested:
42
+ logger.warning_once(
43
+ "Stripping request-level %s=%r: GPU video "
44
+ "backend not configured at startup.",
45
+ key,
46
+ requested,
47
+ )
48
+ runtime_kwargs = {
49
+ k: v for k, v in runtime_kwargs.items() if k != key
50
+ }
51
+
52
merged = super().merge_kwargs(default_kwargs, runtime_kwargs)
53
# fps and num_frames interact with each other, so if either is
54
# overridden at request time, wipe the other from defaults to
vllm/multimodal/video.py
+7
-1
@@ -39,6 +39,7 @@ class VideoLoaderRegistry(ExtensionManager):
39
def __init__(self) -> None:
40
super().__init__()
41
self.processor2backend: dict[str, str] = {}
42
+ self._requires_gpu: dict[str, bool] = {}
43
44
@staticmethod
45
def _normalize_registered_video_processors(
@@ -62,11 +63,13 @@ class VideoLoaderRegistry(ExtensionManager):
63
name: str,
64
*,
65
video_processor: str | tuple[str, ...] | None = None,
66
+ requires_gpu: bool = False,
67
):
68
processors = self._normalize_registered_video_processors(video_processor)
69
70
def wrap(cls_to_register):
71
self.name2class[name] = cls_to_register
72
+ self._requires_gpu[name] = requires_gpu
73
for processor_name in processors:
74
self.processor2backend[processor_name] = name
75
return cls_to_register
@@ -82,6 +85,9 @@ class VideoLoaderRegistry(ExtensionManager):
85
86
return self.processor2backend.get(video_processor)
87
88
+ def backend_requires_gpu(self, name: str) -> bool:
89
+ return self._requires_gpu.get(name, False)
90
+
91
92
def get_video_loader_backend_for_processor(
93
video_processor: str | None,
@@ -909,7 +915,7 @@ class VideoBackend(
915
)
916
917
912
-@VIDEO_LOADER_REGISTRY.register(PYNVVIDEOCODEC_VIDEO_BACKEND)
918
+@VIDEO_LOADER_REGISTRY.register(PYNVVIDEOCODEC_VIDEO_BACKEND, requires_gpu=True)
919
class PyNvVideoCodecVideoBackend(VideoBackend):
920
"""Hardware-accelerated video backend using PyNvVideoCodec.
921
vllm/v1/worker/gpu_worker.py
+6
-6
@@ -55,7 +55,7 @@ from vllm.multimodal.video import (
55
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
56
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
57
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
58
- PYNVVIDEOCODEC_VIDEO_BACKEND,
58
+ VIDEO_LOADER_REGISTRY,
59
)
60
from vllm.platforms import current_platform
61
from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper
@@ -584,15 +584,15 @@ class Worker(WorkerBase):
584
)
585
586
@staticmethod
587
- def _uses_pynvvideocodec_video_backend(mm_config) -> bool:
587
+ def _uses_gpu_video_backend(mm_config) -> bool:
588
video_kwargs = mm_config.media_io_kwargs.get("video", {})
589
video_loader_backend = (
590
video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
591
)
592
codec_backend = video_kwargs.get("backend")
593
- return (
594
- video_loader_backend == PYNVVIDEOCODEC_VIDEO_BACKEND
595
- or codec_backend == PYNVVIDEOCODEC_VIDEO_BACKEND
593
+ return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or (
594
+ codec_backend is not None
595
+ and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend)
596
)
597
598
def _reserve_mm_ipc_gpu_memory(self, available_kv_cache_memory_bytes: int) -> int:
@@ -623,7 +623,7 @@ class Worker(WorkerBase):
623
)
624
decoder_reserved_bytes = (
625
num_api_servers * per_server_decoder_bytes
626
- if self._uses_pynvvideocodec_video_backend(mm_config)
626
+ if self._uses_gpu_video_backend(mm_config)
627
else 0
628
)
629
reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes