[Multimodal] Use GPU NVDEC for EPD encoder-only instance video media IO (#53675)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Isotr0py committed
Sep 11, 2026 at 22:20 UTC
22258a26bc090bccf5473cf681bbe9bac41bd035
12 files changed
+373
-24
tests/config/test_multimodal_config.py
+82
@@ -375,3 +375,85 @@ def test_vllm_config_runs_the_mm_processor_device_check():
375
pytest.raises(ValueError, match="also runs the language model"),
376
):
377
VllmConfig._validate_mm_processor_device(vllm_config)
378
+
379
+
380
+def _resolve_mm_video_decode_device(
381
+ *,
382
+ ec_role: ECRole | None,
383
+ mm_tensor_ipc: str = "torch_shm",
384
+ device: str | None = None,
385
+ video_kwargs: dict | None = None,
386
+ torchcodec_available: bool = True,
387
+) -> dict:
388
+ """Run processor-device then video-decode-device resolution and report
389
+ the resulting video media IO kwargs."""
390
+ mm_config = MultiModalConfig(
391
+ mm_processor_kwargs={} if device is None else {"device": device},
392
+ mm_tensor_ipc=mm_tensor_ipc, # type: ignore[arg-type]
393
+ media_io_kwargs={} if video_kwargs is None else {"video": dict(video_kwargs)},
394
+ )
395
+ model_config = MagicMock(spec=ModelConfig)
396
+ model_config.multimodal_config = mm_config
397
+ vllm_config = MagicMock(spec=VllmConfig)
398
+ vllm_config.model_config = model_config
399
+ vllm_config.ec_transfer_config = (
400
+ None
401
+ if ec_role is None
402
+ else ECTransferConfig(ec_connector="ECExampleConnector", ec_role=ec_role)
403
+ )
404
+
405
+ with (
406
+ patch("vllm.platforms.current_platform.device_type", "cuda"),
407
+ patch(
408
+ "vllm.utils.import_utils.check_torchcodec_available",
409
+ side_effect=None if torchcodec_available else ImportError("torchcodec"),
410
+ ),
411
+ ):
412
+ VllmConfig._resolve_mm_processor_device(vllm_config)
413
+ VllmConfig._resolve_mm_video_decode_device(vllm_config)
414
+ return mm_config.media_io_kwargs.get("video", {})
415
+
416
+
417
+def test_auto_video_decode_uses_nvdec_on_encoder_instance():
418
+ """The processor runs on the accelerator there, so decoded frames should
419
+ stay on-device too."""
420
+ assert _resolve_mm_video_decode_device(ec_role="ec_producer") == {
421
+ "backend": "torchcodec",
422
+ "device": "cuda",
423
+ }
424
+
425
+
426
+@pytest.mark.parametrize("ec_role", [None, "ec_consumer", "ec_both"])
427
+def test_auto_video_decode_stays_on_cpu_off_encoder_instance(
428
+ ec_role: ECRole | None,
429
+):
430
+ """The processor stays on CPU off encode-only instances, so video
431
+ decoding should too."""
432
+ assert _resolve_mm_video_decode_device(ec_role=ec_role) == {}
433
+
434
+
435
+def test_auto_video_decode_follows_explicit_processor_device():
436
+ assert _resolve_mm_video_decode_device(ec_role="ec_producer", device="cpu") == {}
437
+ assert _resolve_mm_video_decode_device(ec_role="ec_producer", device="cuda") == {
438
+ "backend": "torchcodec",
439
+ "device": "cuda",
440
+ }
441
+
442
+
443
+def test_auto_video_decode_respects_explicit_media_io_kwargs():
444
+ assert _resolve_mm_video_decode_device(
445
+ ec_role="ec_producer", video_kwargs={"backend": "opencv"}
446
+ ) == {"backend": "opencv"}
447
+ assert _resolve_mm_video_decode_device(
448
+ ec_role="ec_producer", video_kwargs={"device": "cpu"}
449
+ ) == {"device": "cpu"}
450
+
451
+
452
+def test_auto_video_decode_skipped_without_torchcodec():
453
+ assert (
454
+ _resolve_mm_video_decode_device(
455
+ ec_role="ec_producer",
456
+ torchcodec_available=False,
457
+ )
458
+ == {}
459
+ )
tests/multimodal/media/test_video.py
+44
@@ -532,6 +532,50 @@ class TestMergeKwargsGpuBackendPolicy:
532
def test_unknown_backend_not_treated_as_gpu(self):
533
assert not VIDEO_LOADER_REGISTRY.backend_requires_gpu("totally_unknown")
534
535
+ def test_strips_request_level_device(self):
536
+ """The decode device is a startup-only knob: a request must not move
537
+ decoding onto the GPU when the startup config did not opt in, nor off
538
+ it when it did."""
539
+ result = VideoMediaIO.merge_kwargs(
540
+ default_kwargs={"backend": "torchcodec"},
541
+ runtime_kwargs={"device": "cuda"},
542
+ )
543
+ assert "device" not in result
544
+
545
+ result = VideoMediaIO.merge_kwargs(
546
+ default_kwargs={"backend": "torchcodec", "device": "cuda"},
547
+ runtime_kwargs={"device": "cpu", "num_frames": 8},
548
+ )
549
+ assert result["device"] == "cuda"
550
+ assert result["num_frames"] == 8
551
+
552
+
553
+@pytest.mark.parametrize(
554
+ "default_kwargs",
555
+ [
556
+ {"backend": "torchcodec", "device": "cuda", "seek_mode": "approximate"},
557
+ {"backend": "pynvvideocodec", "hw_decoders": 2},
558
+ ],
559
+)
560
+def test_switching_backend_drops_stale_codec_options(default_kwargs):
561
+ """Codec-specific options from the static config must not leak into a
562
+ different codec backend selected per-request, where they would fail the
563
+ new backend's option validation."""
564
+ result = VideoMediaIO.merge_kwargs(
565
+ default_kwargs={**default_kwargs, "num_frames": 8},
566
+ runtime_kwargs={"backend": "opencv"},
567
+ )
568
+ assert result == {"backend": "opencv", "num_frames": 8}
569
+
570
+
571
+def test_same_backend_keeps_codec_options():
572
+ result = VideoMediaIO.merge_kwargs(
573
+ default_kwargs={"backend": "torchcodec", "device": "cuda"},
574
+ runtime_kwargs={"backend": "torchcodec", "num_frames": 8},
575
+ )
576
+ assert result["device"] == "cuda"
577
+ assert result["num_frames"] == 8
578
+
579
580
@pytest.mark.parametrize("layout", ["nhwc", "nchw"])
581
def test_pynvvc_frames_normalized_to_nhwc(layout: str):
tests/multimodal/test_hasher.py
+27
@@ -137,6 +137,33 @@ def test_hash_collision_video_num_frames():
137
)
138
139
140
+def test_hash_video_tensor_frames():
141
+ """Videos holding tensor frames (e.g. NVDEC-decoded) hash like
142
+ array-framed ones, from the original bytes without a D2H copy."""
143
+ source = b"x" * 100
144
+
145
+ def item_for_hash(frames):
146
+ metadata = {
147
+ "total_num_frames": 2,
148
+ "fps": 2.0,
149
+ "duration": 1.0,
150
+ "video_backend": "torchcodec",
151
+ "frames_indices": [0, 1],
152
+ "do_sample_frames": False,
153
+ }
154
+ video = MediaWithBytes((frames, metadata), source)
155
+ items = MultiModalDataParser()._parse_video_data([video])
156
+ return items.get_all_items_for_hash()[0]
157
+
158
+ np_frames = np.zeros((2, 8, 8, 3), dtype=np.uint8)
159
+ torch_frames = torch.zeros((2, 8, 8, 3), dtype=torch.uint8)
160
+
161
+ hasher = MultiModalHasher
162
+ assert hasher.hash_kwargs("blake3", video=item_for_hash(np_frames)) == (
163
+ hasher.hash_kwargs("blake3", video=item_for_hash(torch_frames))
164
+ )
165
+
166
+
167
def test_hash_non_contiguous_array():
168
arr = np.arange(24).reshape(4, 6).T
169
assert not arr.flags.c_contiguous
tests/multimodal/test_parse.py
+22
@@ -56,6 +56,28 @@ def test_frame_size_hwc_chw(frame):
56
assert items.get_frame_size(0) == (W, H)
57
58
59
+def test_video_with_metadata_tensor_passthrough():
60
+ """Tensor frames pass through unchanged regardless of device: HF video
61
+ processors accept tensors, and device-resident frames (e.g. NVDEC-decoded)
62
+ must not be copied back to host."""
63
+ frames = torch.zeros((4, H, W, 3), dtype=torch.uint8)
64
+ video, metadata = MultiModalDataParser()._get_video_with_metadata(frames)
65
+
66
+ assert video is frames
67
+ assert metadata is None
68
+
69
+
70
+@pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires CUDA")
71
+def test_video_with_metadata_keeps_device_tensor():
72
+ """Device-resident frames (e.g. NVDEC-decoded) pass through as tensors,
73
+ so a device-side HF processor can consume them without a D2H copy."""
74
+ frames = torch.zeros((4, H, W, 3), dtype=torch.uint8, device="cuda")
75
+ video, metadata = MultiModalDataParser()._get_video_with_metadata(frames)
76
+
77
+ assert video is frames
78
+ assert metadata is None
79
+
80
+
81
@pytest.mark.parametrize(
82
"frames",
83
[
tests/multimodal/test_video.py
+37
-2
@@ -11,6 +11,7 @@ from pathlib import Path
11
import numpy as np
12
import numpy.typing as npt
13
import pytest
14
+import torch
15
from transformers import AutoVideoProcessor
16
from transformers.video_utils import VideoMetadata
17
@@ -173,9 +174,14 @@ def test_decode_video_imports_only_selected_backend(
174
[
175
(
176
"torchcodec",
176
- {"min_frames": 4, "num_ffmpeg_threads": 2, "seek_mode": "approximate"},
177
+ {
178
+ "min_frames": 4,
179
+ "num_ffmpeg_threads": 2,
180
+ "seek_mode": "approximate",
181
+ "device": "cuda",
182
+ },
183
{"min_frames": 4},
178
- {"num_ffmpeg_threads": 2, "seek_mode": "approximate"},
184
+ {"num_ffmpeg_threads": 2, "seek_mode": "approximate", "device": "cuda"},
185
),
186
(
187
"deepstream",
@@ -205,6 +211,11 @@ def test_video_backend_rejects_options_for_another_decoder():
211
):
212
resolve_video_backend_kwargs("opencv", {"num_ffmpeg_threads": 2})
213
214
+ with pytest.raises(
215
+ ValueError, match="device is not supported by the 'opencv' backend"
216
+ ):
217
+ resolve_video_backend_kwargs("opencv", {"device": "cuda"})
218
+
219
220
@pytest.mark.parametrize(
221
("backend", "error"),
@@ -1205,6 +1216,30 @@ def test_torchcodec_backend_rejects_frame_recovery(dummy_video_path):
1216
)
1217
1218
1219
+@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
1220
+def test_torchcodec_backend_cuda_decodes_on_gpu(dummy_video_path):
1221
+ """With device="cuda", torchcodec decodes via NVDEC and keeps the frames
1222
+ on the GPU instead of returning a host-side numpy array."""
1223
+ pytest.importorskip("torchcodec")
1224
+
1225
+ with open(dummy_video_path, "rb") as f:
1226
+ video_data = f.read()
1227
+
1228
+ loader = VIDEO_LOADER_REGISTRY.load("opencv")
1229
+ frames, metadata = loader.load_bytes(
1230
+ video_data, num_frames=8, backend="torchcodec", device="cuda"
1231
+ )
1232
+
1233
+ assert isinstance(frames, torch.Tensor)
1234
+ assert frames.device.type == "cuda"
1235
+ assert frames.dtype == torch.uint8
1236
+ assert frames.ndim == 4
1237
+ assert frames.shape[3] == 3 # RGB
1238
+ assert frames.shape[0] == 8
1239
+ assert frames.shape[0] == len(metadata["frames_indices"])
1240
+ assert metadata["video_backend"] == "torchcodec"
1241
+
1242
+
1243
def test_torchcodec_backend_returns_target_frames_not_keyframes():
1244
"""Regression test: torchcodec must return the requested frames, not the
1245
GOP keyframe they seek back to.
vllm/config/vllm.py
+56
@@ -1736,6 +1736,7 @@ class VllmConfig:
1736
1737
self._resolve_mm_embedding_inputs()
1738
self._resolve_mm_processor_device()
1739
+ self._resolve_mm_video_decode_device()
1740
self._validate_mm_processor_device()
1741
1742
if self.use_v2_model_runner:
@@ -2623,6 +2624,61 @@ class VllmConfig:
2624
device_type,
2625
)
2626
2627
+ def _resolve_mm_video_decode_device(self) -> None:
2628
+ """Default video decoding to torchcodec GPU backend for EPD encoder-only
2629
+ instance if the mm processor runs on CUDA.
2630
+
2631
+ The processor consumes the decoded frames on-device in that case, so
2632
+ keeping the frames on the GPU skips the host round-trip through the
2633
+ CPU media path. An explicit codec/backend choice in
2634
+ `--media-io-kwargs` is left alone, and the default is skipped where
2635
+ torchcodec (or its FFmpeg runtime) is unavailable.
2636
+ """
2637
+ if self.model_config is None or self.model_config.multimodal_config is None:
2638
+ return
2639
+ mm_config = self.model_config.multimodal_config
2640
+
2641
+ ec_config = self.ec_transfer_config
2642
+ # An EC producer that is not also a consumer runs no forward pass and
2643
+ # allocates no KV cache, so frontend accelerator work has the device to
2644
+ # itself.
2645
+ if ec_config is None or not ec_config.is_encode_only:
2646
+ return
2647
+
2648
+ from vllm.platforms import current_platform
2649
+
2650
+ device_type = current_platform.device_type
2651
+ if (
2652
+ device_type != "cuda"
2653
+ or mm_config.get_mm_processor_device_type() != device_type
2654
+ ):
2655
+ return
2656
+
2657
+ # User set video backend or device explicitly
2658
+ video_kwargs = mm_config.media_io_kwargs.setdefault("video", {})
2659
+ if "backend" in video_kwargs or "device" in video_kwargs:
2660
+ return
2661
+
2662
+ from vllm.utils.import_utils import check_torchcodec_available
2663
+
2664
+ try:
2665
+ check_torchcodec_available()
2666
+ except (ImportError, RuntimeError):
2667
+ # torchcodec is not installed, or is installed without a usable
2668
+ # FFmpeg runtime (it raises rather than returning False).
2669
+ logger.info_once(
2670
+ "EPD encoder instance: keeping CPU video decoding because "
2671
+ "torchcodec is not available (needs a CUDA build with FFmpeg)."
2672
+ )
2673
+ return
2674
+
2675
+ video_kwargs["backend"] = "torchcodec"
2676
+ video_kwargs["device"] = device_type
2677
+ logger.info_once(
2678
+ "EPD encoder instance: decoding video with NVDEC (torchcodec device=%s).",
2679
+ device_type,
2680
+ )
2681
+
2682
def _validate_mm_processor_device(self) -> None:
2683
"""Hand the EC config to `MultiModalConfig`, which owns the rule."""
2684
model_config = self.model_config
vllm/multimodal/hasher.py
+4
-1
@@ -89,8 +89,11 @@ class MultiModalHasher:
89
)
90
return cls.iter_item_to_bytes("image", obj.original_bytes)
91
92
- if isinstance(obj, MediaWithBytes) and isinstance(obj.media, np.ndarray):
92
+ if isinstance(obj, MediaWithBytes) and isinstance(
93
+ obj.media, (np.ndarray, torch.Tensor)
94
+ ):
95
frames = obj.media
96
+ # Both np.ndarray and torch.Tensor expose .nbytes.
97
if frames.nbytes < len(obj.original_bytes):
98
return cls.iter_item_to_bytes("video", frames)
99
return cls.iter_item_to_bytes("video", obj.original_bytes)
vllm/multimodal/media/video.py
+22
-5
@@ -19,17 +19,23 @@ from vllm.utils.sparse_utils import (
19
safe_to_dense,
20
)
21
22
-from ..video import VIDEO_LOADER_REGISTRY
22
+from ..video import VIDEO_LOADER_REGISTRY, DecodedFrames
23
+from ..video_decoders import incompatible_backend_options
24
from .base import MediaIO, MediaWithBytes
25
from .image import MAGIC_NUMPY_PREFIX, ImageMediaIO
26
27
logger = init_logger(__name__)
28
29
29
-class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]):
30
+class VideoMediaIO(MediaIO[MediaWithBytes[tuple[DecodedFrames, dict[str, Any]]]]):
31
"""Configuration values can be user-provided either by --media-io-kwargs or
32
by the runtime API field "media_io_kwargs". Ensure proper validation and
33
error handling.
34
+
35
+ Use --media-io-kwargs '{"video": {"backend": "torchcodec",
36
+ "device": "cuda"}}' to decode with NVDEC and keep frames on the GPU,
37
+ which pairs with a device-side multi-modal processor
38
+ (--mm-processor-device, EPD encode-only instances).
39
"""
40
41
@classmethod
@@ -43,6 +49,9 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]):
49
runtime_kwargs = dict(runtime_kwargs)
50
runtime_kwargs.pop("hw_decoders", None)
51
runtime_kwargs.pop("pool_size", None)
52
+ # The decode device determines which device the decoded frames
53
+ # (and thus the processor pipeline) live on; startup-only.
54
+ runtime_kwargs.pop("device", None)
55
56
# Block request-level selection of GPU video backends that
57
# were not configured (and VRAM-reserved) at startup.
@@ -70,6 +79,14 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]):
79
merged.pop("fps", None)
80
elif "fps" in runtime_kwargs and "num_frames" not in runtime_kwargs:
81
merged.pop("num_frames", None)
82
+
83
+ requested_backend = runtime_kwargs.get("backend")
84
+ static_backend = (default_kwargs or {}).get("backend")
85
+ if requested_backend and requested_backend != static_backend:
86
+ for name in incompatible_backend_options(
87
+ static_backend, requested_backend
88
+ ):
89
+ merged.pop(name, None)
90
return merged
91
92
def __init__(
@@ -101,7 +118,7 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]):
118
119
def load_bytes(
120
self, data: bytes
104
- ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
121
+ ) -> MediaWithBytes[tuple[DecodedFrames, dict[str, Any]]]:
122
video = self.video_loader.load_bytes(
123
data, num_frames=self.num_frames, **self.kwargs
124
)
@@ -109,7 +126,7 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]):
126
127
def load_base64(
128
self, media_type: str, data: str
112
- ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
129
+ ) -> MediaWithBytes[tuple[DecodedFrames, dict[str, Any]]]:
130
if media_type.lower() == "video/jpeg":
131
load_frame = partial(
132
self.image_io.load_base64,
@@ -177,7 +194,7 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]):
194
195
def load_file(
196
self, filepath: Path
180
- ) -> MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]:
197
+ ) -> MediaWithBytes[tuple[DecodedFrames, dict[str, Any]]]:
198
with filepath.open("rb") as f:
199
data = f.read()
200
vllm/multimodal/parse.py
+9
-7
@@ -36,6 +36,7 @@ from .inputs import (
36
VideoItem,
37
)
38
from .media import MediaWithBytes
39
+from .video import DecodedFrames
40
41
_T = TypeVar("_T")
42
_I = TypeVar("_I")
@@ -653,7 +654,7 @@ class MultiModalDataParser:
654
def _get_video_with_metadata(
655
self,
656
video: VideoItem,
656
- ) -> tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any] | None]:
657
+ ) -> tuple[DecodedFrames | MediaWithBytes[DecodedFrames], dict[str, Any] | None]:
658
if isinstance(video, MediaWithBytes):
659
new_video, metadata = self._get_video_with_metadata(video.media)
660
return MediaWithBytes(new_video, video.original_bytes), metadata
@@ -661,10 +662,11 @@ class MultiModalDataParser:
662
return video
663
if isinstance(video, list):
664
return np.array(video), None
664
- if isinstance(video, np.ndarray):
665
+ if isinstance(video, (np.ndarray, torch.Tensor)):
666
+ # Tensors pass through untouched: HF video processors accept them
667
+ # directly, and device tensors (e.g. NVDEC-decoded frames) must
668
+ # stay on-device to avoid a D2H round-trip.
669
return video, None
666
- if isinstance(video, torch.Tensor):
667
- return video.numpy(), None
670
671
assert_never(video)
672
@@ -769,9 +771,9 @@ class MultiModalDataParser:
771
data_items = data # type: ignore[assignment]
772
773
new_videos = list[
772
- np.ndarray
773
- | MediaWithBytes[np.ndarray]
774
- | tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any]]
774
+ DecodedFrames
775
+ | MediaWithBytes[DecodedFrames]
776
+ | tuple[DecodedFrames | MediaWithBytes[DecodedFrames], dict[str, Any]]
777
| None
778
]()
779
metadata_lst: list[dict[str, Any] | None] = []
vllm/multimodal/video.py
+20
-5
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import math
4
from abc import abstractmethod
5
-from typing import Any, ClassVar, cast
5
+from typing import Any, ClassVar, TypeAlias, cast
6
7
import numpy as np
8
import numpy.typing as npt
@@ -28,6 +28,10 @@ except ImportError:
28
29
logger = init_logger(__name__)
30
31
+DecodedFrames: TypeAlias = npt.NDArray | torch.Tensor
32
+"""Decoded video frames: a host ``np.ndarray``, or a device ``torch.Tensor``
33
+when a GPU decoding codec is used (e.g. torchcodec with ``device="cuda"``)."""
34
+
35
36
class VideoLoaderRegistry(ExtensionManager):
37
def __init__(self) -> None:
@@ -164,8 +168,13 @@ class VideoLoader:
168
cls,
169
data: bytes,
170
**kwargs,
167
- ) -> tuple[npt.NDArray, dict[str, Any]]:
168
- """Load video frames from bytes and return (frames_array, metadata_dict)."""
171
+ ) -> tuple[DecodedFrames, dict[str, Any]]:
172
+ """Load video frames from bytes and return (frames, metadata_dict).
173
+
174
+ ``frames`` is a CPU ``np.ndarray`` unless a GPU decoding codec is
175
+ used (e.g. torchcodec with ``device="cuda"``), in which case it is
176
+ a ``torch.Tensor`` living on that device.
177
+ """
178
raise NotImplementedError
179
180
@classmethod
@@ -239,7 +248,7 @@ class VideoBackend(VideoLoader):
248
*,
249
backend: VideoDecoderBackend = "opencv",
250
**kwargs: Any,
242
- ) -> tuple[npt.NDArray, dict[str, Any]]:
251
+ ) -> tuple[DecodedFrames, dict[str, Any]]:
252
"""Load sampled frames from raw video bytes.
253
254
Args:
@@ -265,6 +274,11 @@ class VideoBackend(VideoLoader):
274
creation at the cost of relying on the file's metadata. See
275
https://meta-pytorch.org/torchcodec/stable/generated_examples/decoding/approximate_mode.html
276
for details.
277
+ - ``device`` (TorchCodec): ``"cpu"`` (default) decodes on the
278
+ host and returns a ``np.ndarray``; ``"cuda"`` decodes with
279
+ NVDEC and returns the frames as a CUDA ``torch.Tensor``,
280
+ which a device-side HF video processor can consume without
281
+ a host round-trip.
282
- ``hw_decoders`` (PyNvVideoCodec): maximum number of
283
concurrent decoder slots. Defaults to 2 and must be a
284
positive integer.
@@ -272,7 +286,8 @@ class VideoBackend(VideoLoader):
286
size and pool acquisition timeout in seconds.
287
288
Returns:
275
- Tuple of ``(frames_array, metadata_dict)``.
289
+ Tuple of ``(frames, metadata_dict)``, where ``frames`` is a
290
+ CPU ``np.ndarray`` unless TorchCodec decodes on ``device="cuda"``.
291
"""
292
target = VideoTargetMetadata(
293
num_frames=num_frames, fps=fps, max_duration=max_duration
vllm/multimodal/video_decoders/__init__.py
+13
@@ -19,6 +19,7 @@ _BACKEND_OPTION_DEFAULTS: dict[str, dict[str, Any]] = {
19
"torchcodec": {
20
"num_ffmpeg_threads": 0,
21
"seek_mode": "exact",
22
+ "device": "cpu",
23
},
24
PYNVVIDEOCODEC_VIDEO_BACKEND: {
25
"hw_decoders": PYNVVIDEOCODEC_DEFAULT_HW_DECODERS,
@@ -40,6 +41,17 @@ def _get_backend_option_defaults(backend: str) -> dict[str, Any]:
41
) from None
42
43
44
+# TODO(Isotr0py): Make kwargs structured.
45
+def incompatible_backend_options(
46
+ old_backend: str | None,
47
+ new_backend: str,
48
+) -> set[str]:
49
+ """Option names accepted by ``old_backend`` but not by ``new_backend``."""
50
+ old = _BACKEND_OPTION_DEFAULTS.get(old_backend or "", {})
51
+ new = _BACKEND_OPTION_DEFAULTS.get(new_backend, {})
52
+ return old.keys() - new.keys()
53
+
54
+
55
def resolve_video_backend_kwargs(
56
backend: str,
57
kwargs: dict[str, Any],
@@ -103,5 +115,6 @@ __all__ = [
115
"VideoTargetMetadata",
116
"check_frame_pixel_limit",
117
"decode_video",
118
+ "incompatible_backend_options",
119
"resolve_video_backend_kwargs",
120
]
vllm/multimodal/video_decoders/torchcodec.py
+37
-4
@@ -5,7 +5,10 @@ from typing import Literal
5
6
import numpy as np
7
import numpy.typing as npt
8
+import torch
9
10
+from vllm.logger import init_logger
11
+from vllm.platforms import current_platform
12
from vllm.utils.import_utils import PlaceholderModule, check_torchcodec_available
13
14
from .base import (
@@ -21,6 +24,8 @@ except (ImportError, RuntimeError):
24
"decoders.VideoDecoder"
25
)
26
27
+logger = init_logger(__name__)
28
+
29
30
def decode_torchcodec(
31
loader_cls,
@@ -30,12 +35,14 @@ def decode_torchcodec(
35
*,
36
num_ffmpeg_threads: int = 0,
37
seek_mode: Literal["exact", "approximate"] = "exact",
33
-) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]:
38
+ device: str = "cpu",
39
+) -> tuple[npt.NDArray | torch.Tensor, VideoSourceMetadata, list[int], list[int]]:
40
check_torchcodec_available()
41
decoder = TorchCodecVideoBackendMixin.make_torchcodec_decoder(
42
data,
43
num_ffmpeg_threads=num_ffmpeg_threads,
44
seek_mode=seek_mode,
45
+ device=device,
46
)
47
check_frame_pixel_limit(
48
decoder.metadata.width or 0,
@@ -48,7 +55,7 @@ def decode_torchcodec(
55
source=source, target=target, **sampling_kwargs
56
)
57
frames, valid = TorchCodecVideoBackendMixin.decode_torchcodec_frames(
51
- decoder, frame_idx
58
+ decoder, frame_idx, device=device
59
)
60
return frames, source, frame_idx, valid
61
@@ -67,7 +74,19 @@ class TorchCodecVideoBackendMixin:
74
*,
75
num_ffmpeg_threads: int = 0,
76
seek_mode: Literal["exact", "approximate"] = "exact",
77
+ device: str = "cpu",
78
) -> "VideoDecoder":
79
+ torch_device = torch.device(device)
80
+ if torch_device.type == "cuda" and not current_platform.is_cuda():
81
+ raise ValueError(
82
+ f"torchcodec video decoding on device {device!r} requires "
83
+ "a CUDA-capable platform."
84
+ )
85
+ elif torch_device.type not in ("cpu", "cuda"):
86
+ raise ValueError(
87
+ f"torchcodec video decoding only supports 'cpu' and 'cuda' "
88
+ f"devices, got {device!r}."
89
+ )
90
# NHWC matches the (num_frames, H, W, 3) uint8 RGB layout the rest
91
# of the pipeline expects, avoiding a transpose.
92
return VideoDecoder(
@@ -75,6 +94,7 @@ class TorchCodecVideoBackendMixin:
94
dimension_order="NHWC",
95
num_ffmpeg_threads=num_ffmpeg_threads,
96
seek_mode=seek_mode,
97
+ device=device,
98
)
99
100
@staticmethod
@@ -91,10 +111,23 @@ class TorchCodecVideoBackendMixin:
111
def decode_torchcodec_frames(
112
decoder: "VideoDecoder",
113
frame_indices: list[int],
94
- ) -> tuple[npt.NDArray, list[int]]:
114
+ *,
115
+ device: str = "cpu",
116
+ ) -> tuple[npt.NDArray | torch.Tensor, list[int]]:
117
"""Decode the requested indices in one batched, index-exact call."""
118
if not frame_indices:
119
return np.empty((0,), dtype=np.uint8), []
120
# Note: torchcodec releases the GIL for the entire call
121
batch = decoder.get_frames_at(frame_indices)
100
- return batch.data.numpy(), list(frame_indices)
122
+ frames = batch.data
123
+ if (
124
+ torch.device(device).type != "cpu"
125
+ and (status := getattr(decoder, "cpu_fallback", None)) is not None
126
+ and bool(status)
127
+ ):
128
+ logger.warning_once(
129
+ "torchcodec could not use NVDEC for this video and "
130
+ "decoded on CPU instead; check codec support and "
131
+ "libnvcuvid."
132
+ )
133
+ return frames, list(frame_indices)