@tduncan / Nvidia-Optimization-Stack-VLLM / commits / e23b19309b

Deepstream video backend (#42424)

Signed-off-by: Viranjan Pagar <vpagar@nvidia.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Roger Wang <hey@rogerw.io>

ViranjanPagar committed Jul 10, 2026 at 14:53 UTC e23b19309b8705b21c3b3ff4129c9974ba15a419

3 files changed +212 -11

docs/features/multimodal_inputs.md

+49

@@ -879,6 +879,55 @@ vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \

879

880 Works with common video formats like MP4 when using OpenCV backends.

881

882 +#### GPU Video Decoding with DeepStream (NVDEC)

883 +

884 +By default vLLM decodes video on the CPU. On NVIDIA GPUs you can instead decode

885 +directly on the hardware video engine (NVDEC) with the DeepStream backend, which

886 +keeps decoding off the CPU and can significantly increase video throughput.

887 +

888 +Install the backend (Linux x86-64 only):

889 +

890 +```bash

891 +pip install vllm[deepstream]

892 +```

893 +

894 +The pip wheel bundles the DeepStream libraries but still relies on a few system

895 +packages that pip cannot install. On Ubuntu:

896 +

897 +```bash

898 +apt-get install -y \

899 + gstreamer1.0-tools gstreamer1.0-plugins-base gstreamer1.0-plugins-good \

900 + gstreamer1.0-plugins-bad gstreamer1.0-libav \

901 + python3-gi python3-gst-1.0 libv4l-0 cuda-libraries-13-0

902 +```

903 +

904 +Select the backend either with an environment variable:

905 +

906 +```bash

907 +export VLLM_VIDEO_LOADER_BACKEND=deepstream

908 +vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct

909 +```

910 +

911 +or per request via `--media-io-kwargs`:

912 +

913 +```bash

914 +vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \

915 + --media-io-kwargs '{"video": {"backend": "deepstream"}}'

916 +```

917 +

918 +**Parameters:**

919 +

920 +- `pool_size`: Number of GPU decode workers in the process-wide decode pool

921 + (clamped to `[1, 16]`). When unset it defaults to

922 + `VLLM_MEDIA_LOADING_THREAD_COUNT` (default `8`). The pool is a singleton, so

923 + the first request's value wins.

924 +

925 +```bash

926 +# Example: 12 decode workers

927 +vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \

928 + --media-io-kwargs '{"video": {"backend": "deepstream", "pool_size": 12}}'

929 +```

930 +

931 #### Pre-extracted Frame Sequences with `media_io_kwargs`

932

933 When you extract video frames on the client side and send them as `video/jpeg` (base64-concatenated JPEG frames), you can preserve the original video metadata by using `media_io_kwargs` in your request. This enables more accurate video understanding by preserving temporal information that would otherwise be lost during client-side frame extraction.

setup.py

@@ -1257,6 +1257,9 @@ setup(

1257 "mistral_common[audio]",

1258 ], # Required for audio processing

1259 "video": [], # Kept for backwards compatibility

1260 + # NVIDIA DeepStream (NVDEC) GPU video-decode backend. Linux x86-64

1261 + # only; also needs system GStreamer + libv4l (see docs).

1262 + "deepstream": ["nvidia-deepstream-videodecode-cu13>=9.0.2"],

1263 "flashinfer": [], # Kept for backwards compatibility

1264 # Optional deps for Helion kernel development

1265 # NOTE: When updating helion version, also update CI files:

vllm/multimodal/video.py

+160 -11

@@ -826,6 +826,114 @@ class PyNvVideoCodecVideoBackendMixin:

826 return frames, source, frame_idx, valid_frame_indices

827

828

829 +class DeepStreamVideoBackendMixin:

830 + """NVIDIA DeepStream (NVDEC) GPU-decode codec utilities.

831 +

832 + Decoding runs on a shared pool of daemon threads inside one CUDA

833 + context (see the ``nvidia-deepstream-videodecode-cu13`` package). The

834 + container bytes are pushed into an ``appsrc`` GStreamer pipeline, so no

835 + local file path is required — HTTP and base64 sources decode identically

836 + to local files.

837 +

838 + Like the OpenCV/PyAV mixins, this provides only the codec layer.

839 + Frame *selection* lives in the loader's

840 + ``compute_frames_index_to_sample`` and arrives here as an explicit

841 + list of frame indices.

842 + """

843 +

844 + # Process-wide lazy decode pool, shared across all DeepStream backends.

845 + _pool: ClassVar[Any] = None

846 + _pool_lock: ClassVar[Any] = None

847 +

848 + @classmethod

849 + def _get_pool(cls, pool_size: int | None = None):

850 + """Lazy-initialize the shared decode pool on first use.

851 +

852 + ``pool_size`` (number of decode worker threads) comes from

853 + ``--media-io-kwargs`` (``{"video": {"pool_size": N}}``); when unset it

854 + defaults to the existing ``VLLM_MEDIA_LOADING_THREAD_COUNT`` so no

855 + DeepStream-specific env var is needed. The pool is a process-wide

856 + singleton, so the first decode's value wins.

857 + """

858 + if cls._pool is not None:

859 + return cls._pool

860 + if cls._pool_lock is None:

861 + cls._pool_lock = threading.Lock()

862 + with cls._pool_lock:

863 + if cls._pool is not None:

864 + return cls._pool

865 + import os

866 +

867 + from nvidia.deepstream_videodecode import DecodePool

868 +

869 + if pool_size is None:

870 + pool_size = int(os.environ.get("VLLM_MEDIA_LOADING_THREAD_COUNT", 8))

871 + pool_size = max(1, min(int(pool_size), 16))

872 + logger.info(

873 + "[DeepStream] initializing decode pool with %d workers",

874 + pool_size,

875 + )

876 + cls._pool = DecodePool(num_workers=pool_size)

877 + return cls._pool

878 +

879 + @classmethod

880 + def decode_indices(

881 + cls,

882 + data: bytes,

883 + frame_indices: list[int],

884 + source: VideoSourceMetadata,

885 + codec: str = "",

886 + pool_size: int | None = None,

887 + timeout_sec: float = 120.0,

888 + ) -> tuple[npt.NDArray, list[int]]:

889 + """Decode the requested frame indices from raw container bytes.

890 +

891 + The whole stream is decoded; the pool keeps exactly the frames whose

892 + decode-order index is in ``frame_indices`` (1:1, frame-exact) and

893 + sends EOS once the last one is matched.

894 +

895 + ``codec`` (e.g. ``"h264"``/``"hevc"``) lets the pool keep its NVDEC

896 + session warm across same-codec streams and rebuild only on a codec

897 + change. Frames are returned as a CPU NHWC uint8 array so the

898 + upstream multimodal parser sees the same shape as the other

899 + backends.

900 + """

901 + if not frame_indices:

902 + raise ValueError("DeepStream backend received no frame indices")

903 +

904 + result = cls._get_pool(pool_size).decode(

905 + data,

906 + target_indices=frame_indices,

907 + codec=codec,

908 + max_frames=len(frame_indices),

909 + timeout_sec=timeout_sec,

910 + )

911 + if result.error:

912 + raise ValueError(f"DeepStream decode failed: {result.error}")

913 + if result.frames is None or result.n_kept == 0:

914 + raise ValueError("DeepStream decode produced no frames")

915 +

916 + valid = frame_indices[: result.n_kept]

917 + # GPU -> CPU NHWC uint8 at the codec boundary (one PCIe copy); keeps

918 + # the array shape identical to the OpenCV/PyAV backends. Copy into

919 + # PINNED host memory (reused across calls by PyTorch's pinned caching

920 + # allocator) so the D2H runs at full PCIe bandwidth (~13 GB/s) rather

921 + # than the ~1 GB/s pageable path that plain ``.cpu()`` takes — ~12x

922 + # faster for a 1080p x8 frame batch (~46ms -> ~4ms). ``numpy()`` keeps

923 + # the pinned tensor alive via the array's base.

924 + import torch

925 +

926 + gpu = result.frames

927 + if gpu.is_cuda:

928 + host = torch.empty(gpu.shape, dtype=gpu.dtype, pin_memory=True)

929 + host.copy_(gpu, non_blocking=True)

930 + torch.cuda.current_stream().synchronize()

931 + arr = host.numpy()

932 + else:

933 + arr = gpu.numpy()

934 + return arr, valid

935 +

936 +

937 @VIDEO_LOADER_REGISTRY.register("opencv")

938 class VideoBackend(

939 VideoLoader,

@@ -833,14 +941,15 @@ class VideoBackend(

941 PyAVVideoBackendMixin,

942 TorchCodecVideoBackendMixin,

943 PyNvVideoCodecVideoBackendMixin,

944 + DeepStreamVideoBackendMixin,

945 ):

946 """Uniform-sampling video backend.

947

948 Samples ``num_frames`` uniformly across the video (or one frame every

949 ``1/fps`` seconds, whichever produces fewer frames). The decoding codec

950 is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``,

842 - ``"torchcodec"`` or ``"pynvvideocodec"``), which can be passed through

843 - ``--media-io-kwargs``. Defaults to ``"opencv"``.

951 + ``"torchcodec"``, ``"pynvvideocodec"``, or ``"deepstream"``),

952 + which can be passed through ``--media-io-kwargs``. Defaults to ``"opencv"``.

953 """

954

955 _sampling_suffix: ClassVar[str] = ""

@@ -885,7 +994,9 @@ class VideoBackend(

994 max_duration: int = 300,

995 frame_recovery: bool = False,

996 *,

888 - backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",

997 + backend: Literal[

998 + "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"

999 + ] = "opencv",

1000 num_ffmpeg_threads: int = 0,

1001 seek_mode: Literal["exact", "approximate"] = "exact",

1002 **kwargs,

@@ -901,7 +1012,7 @@ class VideoBackend(

1012 frame_recovery: Enable forward-scan recovery for failed frames.

1013 Only honored by the OpenCV codec.

1014 backend: Decoding codec — ``"opencv"``, ``"pyav"``,

904 - ``"torchcodec"`` or ``"pynvvideocodec"``.

1015 + ``"torchcodec"``, ``"pynvvideocodec"`` or ``"deepstream"``.

1016 num_ffmpeg_threads: Number of FFmpeg decoding threads, only used by

1017 TorchCodec: ``0`` (default) relies on the FFmpeg default value

1018 which is ``min(cpu_count + 1, 16)``.

@@ -982,11 +1093,37 @@ class VideoBackend(

1093 target,

1094 **kwargs,

1095 )

1096 + elif backend == "deepstream":

1097 + assert not frame_recovery, (

1098 + "frame_recovery is only available for `opencv` backend"

1099 + )

1100 + # Decode-pool size comes from media-io-kwargs (no env var); the

1101 + # pool is a process-wide singleton so the first decode's value

1102 + # wins. Pop it so it isn't forwarded to the frame sampler.

1103 + pool_size = kwargs.pop("pool_size", None)

1104 + # Probe container metadata from the bytes via GStreamer (in

1105 + # the deepstream video-decode wheel) — no PyAV/pymediainfo, no path.

1106 + from nvidia.deepstream_videodecode import probe_metadata

1107 +

1108 + total_frames, original_fps, duration, _w, _h, codec = probe_metadata(data)

1109 + source = cls._prepare_source(

1110 + VideoSourceMetadata(

1111 + total_frames_num=total_frames,

1112 + original_fps=original_fps,

1113 + duration=duration,

1114 + )

1115 + )

1116 + frame_idx = cls.compute_frames_index_to_sample(

1117 + source=source, target=target, **kwargs

1118 + )

1119 + frames, valid = cls.decode_indices(

1120 + data, frame_idx, source, codec=codec, pool_size=pool_size

1121 + )

1122 else:

1123 raise ValueError(

1124 f"Unknown video codec backend {backend!r}; "

1125 "valid options: 'opencv', 'pyav', 'torchcodec', "

989 - "'pynvvideocodec'."

1126 + "'pynvvideocodec' and 'deepstream'."

1127 )

1128

1129 if len(valid) < len(frame_idx):

@@ -1073,7 +1210,9 @@ class Qwen3VLVideoBackend(VideoBackend):

1210 max_duration: int = 300,

1211 frame_recovery: bool = False,

1212 *,

1076 - backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",

1213 + backend: Literal[

1214 + "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"

1215 + ] = "opencv",

1216 **kwargs,

1217 ) -> tuple[npt.NDArray, dict[str, Any]]:

1218 return super().load_bytes(

@@ -1152,7 +1291,9 @@ class Qwen2VLVideoBackend(VideoBackend):

1291 max_duration: int = 300,

1292 frame_recovery: bool = False,

1293 *,

1155 - backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",

1294 + backend: Literal[

1295 + "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"

1296 + ] = "opencv",

1297 **kwargs,

1298 ) -> tuple[npt.NDArray, dict[str, Any]]:

1299 return super().load_bytes(

@@ -1244,7 +1385,9 @@ class DynamicVideoBackend(VideoBackend):

1385 max_duration: int = 300,

1386 frame_recovery: bool = False,

1387 *,

1247 - backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",

1388 + backend: Literal[

1389 + "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"

1390 + ] = "opencv",

1391 **kwargs,

1392 ) -> tuple[npt.NDArray, dict[str, Any]]:

1393 return super().load_bytes(

@@ -1369,7 +1512,9 @@ class GLM46VVideoBackend(VideoBackend):

1512 max_duration: int = 300,

1513 frame_recovery: bool = False,

1514 *,

1372 - backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",

1515 + backend: Literal[

1516 + "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"

1517 + ] = "opencv",

1518 **kwargs,

1519 ) -> tuple[npt.NDArray, dict[str, Any]]:

1520 return super().load_bytes(

@@ -1467,7 +1612,9 @@ class GLMGAVideoBackend(VideoBackend):

1612 max_duration: int = 300,

1613 frame_recovery: bool = False,

1614 *,

1470 - backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",

1615 + backend: Literal[

1616 + "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"

1617 + ] = "opencv",

1618 **kwargs,

1619 ) -> tuple[npt.NDArray, dict[str, Any]]:

1620 frames, metadata = super().load_bytes(

@@ -1790,7 +1937,9 @@ class NemotronVLVideoBackend(VideoBackend):

1937 max_duration: int = 300,

1938 frame_recovery: bool = False,

1939 *,

1793 - backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",

1940 + backend: Literal[

1941 + "opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"

1942 + ] = "opencv",

1943 **kwargs,

1944 ) -> tuple[npt.NDArray, dict[str, Any]]:

1945 frames, metadata = super().load_bytes(