Add TorchCodec as a video decoding backend (#46609)
Signed-off-by: Nicolas Hug <contact@nicolas-hug.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Nicolas Hug committed
Jul 7, 2026 at 03:58 UTC
700e882eab7599ef59c42eb11d26d9879044d059
9 files changed
+300
-12
docs/features/multimodal_inputs.md
+38
@@ -816,6 +816,44 @@ Full example: [examples/generate/multimodal/openai_chat_completion_client_for_mu
816
export VLLM_VIDEO_FETCH_TIMEOUT=<timeout>
817
```
818
819
+#### Video Decoding Backend
820
+
821
+vLLM decodes video bytes into frames using a selectable decoding backend. Three
822
+backends are supported:
823
+
824
+- `opencv` (default): OpenCV-based decoder.
825
+- `pyav`: PyAV decoder.
826
+- `torchcodec`: TorchCodec (PyTorch-native) decoder.
827
+
828
+All three backends are ultimately backed by FFmpeg. `torchcodec` lets
829
+you choose which FFmpeg version is used while `opencv` and `pyav` rely on
830
+whichever FFmpeg build they were linked against.
831
+
832
+Select the backend by passing the `backend` parameter via `--media-io-kwargs`:
833
+
834
+```bash
835
+vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \
836
+ --media-io-kwargs '{"video": {"backend": "torchcodec"}}'
837
+```
838
+
839
+**TorchCodec-specific parameters:**
840
+
841
+The following parameters only apply to the `torchcodec` backend:
842
+
843
+- `num_ffmpeg_threads`: Number of FFmpeg decoding threads. `0` (default) relies
844
+ on the FFmpeg default, which is `min(cpu_count + 1, 16)`. This allows you to
845
+ control thread over-subscription.
846
+- `seek_mode`: Seek mode for the decoder. `"exact"` (default) guarantees
847
+ frame-accurate sampling by scanning the file when the decoder is created.
848
+ `"approximate"` skips that scan for faster decoder creation, at the cost of
849
+ relying on the file's metadata (which may yield less accurate seeking).
850
+
851
+```bash
852
+# Example: TorchCodec with approximate seek mode and 4 FFmpeg threads
853
+vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \
854
+ --media-io-kwargs '{"video": {"backend": "torchcodec", "seek_mode": "approximate", "num_ffmpeg_threads": 4}}'
855
+```
856
+
857
#### Video Frame Recovery
858
859
For improved robustness when processing potentially corrupted or truncated video files, vLLM supports optional frame recovery using a dynamic window forward-scan approach. When enabled, if a target frame fails to load during sequential reading, the next successfully grabbed frame (before the next target frame) will be used in its place.
requirements/cpu.txt
+3
@@ -15,6 +15,9 @@ torchaudio; platform_machine != "s390x" and platform_machine != "riscv64"
15
# required for the image processor of phi3v, this must be updated alongside torch
16
torchvision; platform_machine != "s390x" and platform_machine != "riscv64"
17
18
+# required for the torchcodec video decoding backend
19
+torchcodec >= 0.14; platform_machine != "s390x" and platform_machine != "riscv64" and platform_machine != "ppc64le"
20
+
21
# Intel Extension for PyTorch, only for x86_64 CPUs
22
intel-openmp==2024.2.1; platform_machine == "x86_64"
23
requirements/cuda.txt
+1
@@ -8,6 +8,7 @@ torch==2.11.0
8
torchaudio==2.11.0
9
# These must be updated alongside torch
10
torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
11
+torchcodec >= 0.14
12
PyNvVideoCodec==2.1.0
13
# FlashInfer should be updated together with the Dockerfile
14
flashinfer-python==0.6.13
requirements/test/cpu.txt
+2
@@ -1134,6 +1134,8 @@ torchaudio==2.11.0+cpu
1134
# -r requirements/test/cuda.in
1135
# encodec
1136
# vocos
1137
+torchcodec==0.14.0+cpu
1138
+ # via -r requirements/test/cuda.in
1139
torchvision==0.26.0+cpu
1140
# via
1141
# -r requirements/test/cuda.in
requirements/test/cuda.in
+1
@@ -13,6 +13,7 @@ pytest-cov
13
# testing utils
14
albumentations # required for Nemotron Parse in test_common.py
15
av # required for audio_in_video tests
16
+torchcodec >= 0.14 # required for torchcodec video backend tests
17
backoff # required for phi4mm test
18
blobfile # required for kimi-vl test
19
httpx
requirements/test/cuda.txt
+4
@@ -1233,6 +1233,10 @@ torchaudio==2.11.0+cu130
1233
# -r requirements/test/cuda.in
1234
# encodec
1235
# vocos
1236
+torchcodec==0.14.0+cu130
1237
+ # via
1238
+ # -c requirements/cuda.txt
1239
+ # -r requirements/test/cuda.in
1240
torchvision==0.26.0+cu130
1241
# via
1242
# -c requirements/cuda.txt
requirements/xpu.txt
+1
@@ -15,6 +15,7 @@ numba == 0.65.0 # Required for N-gram speculative decoding
15
torch==2.12.0
16
torchaudio
17
torchvision
18
+torchcodec >= 0.14 # Required for the torchcodec video decoding backend
19
20
auto_round_lib>=0.14.0
21
vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.10.1/vllm_xpu_kernels-0.1.10.1-cp38-abi3-manylinux_2_28_x86_64.whl
tests/multimodal/test_video.py
+150
@@ -768,6 +768,118 @@ def test_pyav_backend_returns_target_frames_not_keyframes():
768
)
769
770
771
+# ============================================================================
772
+# TorchCodec Backend Tests
773
+# ============================================================================
774
+
775
+
776
+def test_torchcodec_backend_loads_frames(
777
+ dummy_video_path, monkeypatch: pytest.MonkeyPatch
778
+):
779
+ """Test that the torchcodec codec backend can load frames."""
780
+ pytest.importorskip("torchcodec")
781
+ with monkeypatch.context() as m:
782
+ m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
783
+
784
+ with open(dummy_video_path, "rb") as f:
785
+ video_data = f.read()
786
+
787
+ loader = VIDEO_LOADER_REGISTRY.load("opencv")
788
+ frames, metadata = loader.load_bytes(
789
+ video_data, num_frames=8, backend="torchcodec"
790
+ )
791
+
792
+ assert frames.ndim == 4
793
+ assert frames.shape[3] == 3 # RGB
794
+ assert frames.shape[0] == 8
795
+ assert frames.shape[0] == len(metadata["frames_indices"])
796
+ assert metadata["video_backend"] == "torchcodec"
797
+ assert "total_num_frames" in metadata
798
+ assert "fps" in metadata
799
+ assert "duration" in metadata
800
+
801
+
802
+def test_torchcodec_dynamic_backend_loads_frames(
803
+ dummy_video_path, monkeypatch: pytest.MonkeyPatch
804
+):
805
+ """Test that the torchcodec codec with dynamic sampling can load frames."""
806
+ pytest.importorskip("torchcodec")
807
+ with monkeypatch.context() as m:
808
+ m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic")
809
+
810
+ with open(dummy_video_path, "rb") as f:
811
+ video_data = f.read()
812
+
813
+ loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
814
+ frames, metadata = loader.load_bytes(
815
+ video_data, fps=2, max_duration=10, backend="torchcodec"
816
+ )
817
+
818
+ assert frames.ndim == 4
819
+ assert frames.shape[3] == 3 # RGB
820
+ assert frames.shape[0] > 0
821
+ assert frames.shape[0] == len(metadata["frames_indices"])
822
+ assert metadata["video_backend"] == "torchcodec_dynamic"
823
+
824
+
825
+def test_torchcodec_backend_rejects_frame_recovery(dummy_video_path):
826
+ """frame_recovery is OpenCV-only; torchcodec must reject it."""
827
+ pytest.importorskip("torchcodec")
828
+ with open(dummy_video_path, "rb") as f:
829
+ video_data = f.read()
830
+
831
+ loader = VIDEO_LOADER_REGISTRY.load("opencv")
832
+ with pytest.raises(AssertionError):
833
+ loader.load_bytes(
834
+ video_data, num_frames=8, backend="torchcodec", frame_recovery=True
835
+ )
836
+
837
+
838
+def test_torchcodec_backend_returns_target_frames_not_keyframes():
839
+ """Regression test: torchcodec must return the requested frames, not the
840
+ GOP keyframe they seek back to.
841
+
842
+ Mirrors ``test_pyav_backend_returns_target_frames_not_keyframes``: a long
843
+ GOP (single keyframe at frame 0) with a per-frame green-channel marker.
844
+ With ``seek_mode="exact"`` torchcodec resolves each index to the exact
845
+ frame, so the returned markers must be distinct, ordered, and match the
846
+ requested indices.
847
+ """
848
+ pytest.importorskip("torchcodec")
849
+ num_frames = 50
850
+ num_sampled = 4
851
+ height, width = 64, 64
852
+
853
+ video_bytes = create_long_gop_video(
854
+ num_frames=num_frames, width=width, height=height
855
+ )
856
+
857
+ loader = VIDEO_LOADER_REGISTRY.load("opencv")
858
+ frames, metadata = loader.load_bytes(
859
+ video_bytes, num_frames=num_sampled, backend="torchcodec"
860
+ )
861
+ assert frames.shape == (num_sampled, height, width, 3)
862
+
863
+ requested = list(metadata["frames_indices"])
864
+ assert len(requested) == num_sampled
865
+
866
+ actual = [int(f[height // 2, width // 2, 1]) for f in frames]
867
+
868
+ assert len(set(actual)) == num_sampled, (
869
+ f"torchcodec returned only {len(set(actual))} distinct frames for "
870
+ f"{num_sampled} requested indices: markers={actual}, "
871
+ f"requested={requested}. Keyframe-snap regression."
872
+ )
873
+
874
+ assert actual == sorted(actual), f"Returned frames out of order: markers={actual}"
875
+
876
+ for marker, want_idx in zip(actual, requested):
877
+ assert abs(marker - want_idx) <= 10, (
878
+ f"Frame mismatch: requested index {want_idx}, "
879
+ f"got marker {marker} (tolerance ±10)"
880
+ )
881
+
882
+
883
@pytest.mark.parametrize(
884
"loader_key, kwargs, expected_num_frames",
885
[
@@ -854,6 +966,42 @@ def test_pyav_backend_returns_target_frames_not_keyframes():
966
120,
967
id="glm46v-pyav-60s",
968
),
969
+ # uniform sampling + torchcodec codec (same frame counts as opencv)
970
+ pytest.param(
971
+ "opencv",
972
+ {"num_frames": 32, "backend": "torchcodec"},
973
+ 32,
974
+ id="torchcodec-num_frames",
975
+ ),
976
+ pytest.param(
977
+ "opencv", {"fps": 2, "backend": "torchcodec"}, 120, id="torchcodec-fps"
978
+ ),
979
+ pytest.param(
980
+ "opencv",
981
+ {"num_frames": 500, "fps": 2, "backend": "torchcodec"},
982
+ 120,
983
+ id="torchcodec-num_frames_wins_fps",
984
+ ),
985
+ # dynamic sampling + torchcodec codec
986
+ pytest.param(
987
+ "opencv_dynamic",
988
+ {"fps": 1, "max_duration": 60, "backend": "torchcodec"},
989
+ 60,
990
+ id="torchcodec_dynamic-within_max_duration",
991
+ ),
992
+ pytest.param(
993
+ "opencv_dynamic",
994
+ {"fps": 2, "max_duration": 30, "backend": "torchcodec"},
995
+ 60,
996
+ id="torchcodec_dynamic-exceeds_max_duration",
997
+ ),
998
+ # glm46v dynamic FPS + torchcodec codec
999
+ pytest.param(
1000
+ "glm46v",
1001
+ {"backend": "torchcodec"},
1002
+ 120,
1003
+ id="glm46v-torchcodec-60s",
1004
+ ),
1005
],
1006
)
1007
def test_video_loader_frames_sampling(
@@ -864,6 +1012,8 @@ def test_video_loader_frames_sampling(
1012
expected_num_frames: int,
1013
):
1014
"""Test video loader frames sampling functionality."""
1015
+ if kwargs.get("backend") == "torchcodec":
1016
+ pytest.importorskip("torchcodec")
1017
monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", loader_key)
1018
loader = VIDEO_LOADER_REGISTRY.load(loader_key)
1019
vllm/multimodal/video.py
+100
-12
@@ -31,6 +31,13 @@ try:
31
except ImportError:
32
av = PlaceholderModule("av") # type: ignore[assignment]
33
34
+try:
35
+ from torchcodec.decoders import VideoDecoder
36
+except ImportError:
37
+ VideoDecoder = PlaceholderModule("torchcodec").placeholder_attr( # type: ignore[assignment]
38
+ "decoders.VideoDecoder"
39
+ )
40
+
41
42
logger = init_logger(__name__)
43
@@ -568,6 +575,53 @@ class PyAVVideoBackendMixin:
575
return np.stack(frames_list), valid_indices
576
577
578
+class TorchCodecVideoBackendMixin:
579
+ """TorchCodec (FFmpeg-backed, PyTorch-native) codec utilities.
580
+
581
+ Builds a :class:`~torchcodec.decoders.VideoDecoder` over the in-memory
582
+ bytes and extracts the sampled indices with a single batched
583
+ ``get_frames_at`` call, while releasing the GIL during decode.
584
+ """
585
+
586
+ @staticmethod
587
+ def make_torchcodec_decoder(
588
+ data: bytes,
589
+ *,
590
+ num_ffmpeg_threads: int = 0,
591
+ seek_mode: Literal["exact", "approximate"] = "exact",
592
+ ) -> "VideoDecoder":
593
+ # NHWC matches the (num_frames, H, W, 3) uint8 RGB layout the rest
594
+ # of the pipeline expects, avoiding a transpose.
595
+ return VideoDecoder(
596
+ data,
597
+ dimension_order="NHWC",
598
+ num_ffmpeg_threads=num_ffmpeg_threads,
599
+ seek_mode=seek_mode,
600
+ )
601
+
602
+ @staticmethod
603
+ def get_torchcodec_metadata(decoder: "VideoDecoder") -> VideoSourceMetadata:
604
+ md = decoder.metadata
605
+ total_frames = md.num_frames or 0
606
+ fps = float(md.average_fps) if md.average_fps else 0.0
607
+ duration = float(md.duration_seconds) if md.duration_seconds else 0.0
608
+ if total_frames == 0 and duration > 0 and fps > 0:
609
+ total_frames = int(duration * fps)
610
+ return VideoSourceMetadata(total_frames, fps, duration)
611
+
612
+ @staticmethod
613
+ def decode_torchcodec_frames(
614
+ decoder: "VideoDecoder",
615
+ frame_indices: list[int],
616
+ ) -> tuple[npt.NDArray, list[int]]:
617
+ """Decode the requested indices in one batched, index-exact call."""
618
+ if not frame_indices:
619
+ return np.empty((0,), dtype=np.uint8), []
620
+ # Note: torchcodec releases the GIL for the entire call
621
+ batch = decoder.get_frames_at(frame_indices)
622
+ return batch.data.numpy(), list(frame_indices)
623
+
624
+
625
class PyNvVideoCodecVideoBackendMixin:
626
"""PyNvVideoCodec utilities for GPU-backed frame decode."""
627
@@ -777,14 +831,15 @@ class VideoBackend(
831
VideoLoader,
832
OpenCVVideoBackendMixin,
833
PyAVVideoBackendMixin,
834
+ TorchCodecVideoBackendMixin,
835
PyNvVideoCodecVideoBackendMixin,
836
):
837
"""Uniform-sampling video backend.
838
839
Samples ``num_frames`` uniformly across the video (or one frame every
840
``1/fps`` seconds, whichever produces fewer frames). The decoding codec
786
- is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``, or
787
- ``"pynvvideocodec"``), which can be passed through
841
+ is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``,
842
+ ``"torchcodec"`` or ``"pynvvideocodec"``), which can be passed through
843
``--media-io-kwargs``. Defaults to ``"opencv"``.
844
"""
845
@@ -830,7 +885,9 @@ class VideoBackend(
885
max_duration: int = 300,
886
frame_recovery: bool = False,
887
*,
833
- backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
888
+ backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
889
+ num_ffmpeg_threads: int = 0,
890
+ seek_mode: Literal["exact", "approximate"] = "exact",
891
**kwargs,
892
) -> tuple[npt.NDArray, dict[str, Any]]:
893
"""Load sampled frames from raw video bytes.
@@ -843,8 +900,20 @@ class VideoBackend(
900
dynamic subclass; ignored here.
901
frame_recovery: Enable forward-scan recovery for failed frames.
902
Only honored by the OpenCV codec.
846
- backend: Decoding codec — ``"opencv"``, ``"pyav"``, or
847
- ``"pynvvideocodec"``.
903
+ backend: Decoding codec — ``"opencv"``, ``"pyav"``,
904
+ ``"torchcodec"`` or ``"pynvvideocodec"``.
905
+ num_ffmpeg_threads: Number of FFmpeg decoding threads, only used by
906
+ TorchCodec: ``0`` (default) relies on the FFmpeg default value
907
+ which is ``min(cpu_count + 1, 16)``.
908
+ OpenCV will always use ``min(cpu_count, 16)`` while pyav will
909
+ always use ``min(cpu_count, (height + 15) / 16)``.
910
+ seek_mode: Seek mode for the TorchCodec decoder, only used by
911
+ TorchCodec: ``"exact"`` (default) guarantees frame-accurate
912
+ sampling by scanning the file on creation, while
913
+ ``"approximate"`` skips that scan for faster decoder creation
914
+ at the cost of relying on the file's metadata. See
915
+ https://meta-pytorch.org/torchcodec/stable/generated_examples/decoding/approximate_mode.html
916
+ for details.
917
918
Returns:
919
Tuple of ``(frames_array, metadata_dict)``.
@@ -883,6 +952,24 @@ class VideoBackend(
952
frames, valid = cls.decode_frames(
953
container, frame_idx, source.original_fps, source.duration
954
)
955
+ elif backend == "torchcodec":
956
+ assert not frame_recovery, (
957
+ "frame_recovery is only available for `opencv` backend"
958
+ )
959
+ decoder = cls.make_torchcodec_decoder(
960
+ data,
961
+ num_ffmpeg_threads=num_ffmpeg_threads,
962
+ seek_mode=seek_mode,
963
+ )
964
+ _check_frame_pixel_limit(
965
+ decoder.metadata.width or 0,
966
+ decoder.metadata.height or 0,
967
+ )
968
+ source = cls._prepare_source(cls.get_torchcodec_metadata(decoder))
969
+ frame_idx = cls.compute_frames_index_to_sample(
970
+ source=source, target=target, **kwargs
971
+ )
972
+ frames, valid = cls.decode_torchcodec_frames(decoder, frame_idx)
973
elif backend == PYNVVIDEOCODEC_VIDEO_BACKEND:
974
if frame_recovery:
975
raise ValueError(
@@ -897,7 +984,8 @@ class VideoBackend(
984
else:
985
raise ValueError(
986
f"Unknown video codec backend {backend!r}; "
900
- "valid options: 'opencv', 'pyav', 'pynvvideocodec'."
987
+ "valid options: 'opencv', 'pyav', 'torchcodec', "
988
+ "'pynvvideocodec'."
989
)
990
991
if len(valid) < len(frame_idx):
@@ -984,7 +1072,7 @@ class Qwen3VLVideoBackend(VideoBackend):
1072
max_duration: int = 300,
1073
frame_recovery: bool = False,
1074
*,
987
- backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
1075
+ backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
1076
**kwargs,
1077
) -> tuple[npt.NDArray, dict[str, Any]]:
1078
return super().load_bytes(
@@ -1063,7 +1151,7 @@ class Qwen2VLVideoBackend(VideoBackend):
1151
max_duration: int = 300,
1152
frame_recovery: bool = False,
1153
*,
1066
- backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
1154
+ backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
1155
**kwargs,
1156
) -> tuple[npt.NDArray, dict[str, Any]]:
1157
return super().load_bytes(
@@ -1155,7 +1243,7 @@ class DynamicVideoBackend(VideoBackend):
1243
max_duration: int = 300,
1244
frame_recovery: bool = False,
1245
*,
1158
- backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
1246
+ backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
1247
**kwargs,
1248
) -> tuple[npt.NDArray, dict[str, Any]]:
1249
return super().load_bytes(
@@ -1280,7 +1368,7 @@ class GLM46VVideoBackend(VideoBackend):
1368
max_duration: int = 300,
1369
frame_recovery: bool = False,
1370
*,
1283
- backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
1371
+ backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
1372
**kwargs,
1373
) -> tuple[npt.NDArray, dict[str, Any]]:
1374
return super().load_bytes(
@@ -1378,7 +1466,7 @@ class GLMGAVideoBackend(VideoBackend):
1466
max_duration: int = 300,
1467
frame_recovery: bool = False,
1468
*,
1381
- backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
1469
+ backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
1470
**kwargs,
1471
) -> tuple[npt.NDArray, dict[str, Any]]:
1472
frames, metadata = super().load_bytes(
@@ -1701,7 +1789,7 @@ class NemotronVLVideoBackend(VideoBackend):
1789
max_duration: int = 300,
1790
frame_recovery: bool = False,
1791
*,
1704
- backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
1792
+ backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
1793
**kwargs,
1794
) -> tuple[npt.NDArray, dict[str, Any]]:
1795
frames, metadata = super().load_bytes(