@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 94682b79f4

[multimodal] Make PyNvVideoCodec decoder concurrency configurable (#49753)

Signed-off-by: Brandon Pelfrey <bpelfrey@nvidia.com> Co-authored-by: OpenAI Codex <noreply@openai.com>

Brandon Pelfrey committed Jul 24, 2026 at 23:31 UTC 94682b79f40239369bcefdbebee683a6bed90c85

7 files changed +190 -24

docs/features/multimodal_inputs.md

+21 -4

@@ -818,16 +818,18 @@ Full example: [examples/generate/multimodal/openai_chat_completion_client_for_mu

818

819 #### Video Decoding Backend

820

821 -vLLM decodes video bytes into frames using a selectable decoding backend. Three

821 +vLLM decodes video bytes into frames using a selectable decoding backend. Five

822 backends are supported:

823

824 - `opencv` (default): OpenCV-based decoder.

825 - `pyav`: PyAV decoder.

826 - `torchcodec`: TorchCodec (PyTorch-native) decoder.

827 +- `pynvvideocodec`: NVIDIA NVDEC-based decoder.

828 +- `deepstream`: NVIDIA DeepStream NVDEC-based decoder.

829

828 -All three backends are ultimately backed by FFmpeg. `torchcodec` lets

829 -you choose which FFmpeg version is used while `opencv` and `pyav` rely on

830 -whichever FFmpeg build they were linked against.

830 +The CPU backends are backed by FFmpeg. `torchcodec` lets you choose which FFmpeg

831 +version is used while `opencv` and `pyav` rely on whichever FFmpeg build they

832 +were linked against.

833

834 Select the backend by passing the `backend` parameter via `--media-io-kwargs`:

835

@@ -854,6 +856,21 @@ vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \

856 --media-io-kwargs '{"video": {"backend": "torchcodec", "seek_mode": "approximate", "num_ffmpeg_threads": 4}}'

857 ```

858

859 +**PyNvVideoCodec-specific parameters:**

860 +

861 +- `hw_decoders`: Maximum number of concurrent hardware decoder slots retained

862 + by each API server process. It must be a positive integer and defaults to `2`,

863 + which is the recommended starting point for concurrent video workloads.

864 + Because vLLM reserves GPU memory for these slots at startup, this value cannot

865 + be overridden per request. Benchmark before increasing it because each

866 + additional slot increases the GPU memory reservation.

867 +

868 +```bash

869 +# Example: explicitly use the recommended 2 hardware decoders

870 +vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \

871 + --media-io-kwargs '{"video": {"backend": "pynvvideocodec", "hw_decoders": 2}}'

872 +```

873 +

874 #### Video Frame Recovery

875

876 For improved robustness when processing potentially corrupted or truncated video files, vLLM supports optional frame recovery using a dynamic window forward-scan approach. When enabled, if a target frame fails to load during sequential reading, the next successfully grabbed frame (before the next target frame) will be used in its place.

tests/multimodal/media/test_video.py

+17

@@ -404,6 +404,23 @@ class TestMergeKwargsGpuBackendPolicy:

404 )

405 assert result["backend"] == "pynvvideocodec"

406

407 + def test_strips_request_level_hw_decoders_when_not_static(self):

408 + result = VideoMediaIO.merge_kwargs(

409 + default_kwargs={"video_backend": "pynvvideocodec"},

410 + runtime_kwargs={"hw_decoders": 4},

411 + )

412 + assert "hw_decoders" not in result

413 +

414 + def test_prevents_request_level_hw_decoders_override(self):

415 + result = VideoMediaIO.merge_kwargs(

416 + default_kwargs={

417 + "video_backend": "pynvvideocodec",

418 + "hw_decoders": 2,

419 + },

420 + runtime_kwargs={"hw_decoders": 4},

421 + )

422 + assert result["hw_decoders"] == 2

423 +

424 @pytest.mark.parametrize("backend", ["opencv", "pyav", "torchcodec"])

425 def test_software_video_backend_passes_through(self, backend: str):

426 result = VideoMediaIO.merge_kwargs(

tests/multimodal/test_gpu_ipc_memory.py

+31 -4

@@ -18,7 +18,6 @@ from vllm.multimodal.gpu_ipc_memory import (

18 from vllm.multimodal.video import (

19 PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,

20 PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,

21 - PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,

21 PYNVVIDEOCODEC_VIDEO_BACKEND,

22 )

23 from vllm.utils.mem_constants import GiB_bytes

@@ -28,17 +27,26 @@ def _mm_config(

27 *,

28 mm_ipc_gpu_memory_gb: float = 0,

29 video_backend: str | None = None,

30 + hw_decoders: int | None = None,

31 ) -> MultiModalConfig:

32 - video_kwargs = {} if video_backend is None else {"video_backend": video_backend}

32 + video_kwargs: dict[str, object] = (

33 + {} if video_backend is None else {"video_backend": video_backend}

34 + )

35 + if hw_decoders is not None:

36 + video_kwargs["hw_decoders"] = hw_decoders

37 +

38 return MultiModalConfig(

39 mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,

40 media_io_kwargs={"video": video_kwargs} if video_kwargs else {},

41 )

39 -def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:

44 +def _pynvvideocodec_decoder_budget(

45 + api_process_count: int = 1,

46 + hw_decoders: int = 2,

47 +) -> int:

48 return api_process_count * (

41 - PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS

49 + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * hw_decoders

50 + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES

51 )

@@ -240,3 +248,22 @@ def test_reserve_mm_ipc_gpu_memory_scales_decoder_budget_by_api_servers(

248 _mm_config(),

249 api_process_count=3,

250 ) == available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)

251 +

252 +

253 +def test_reserve_mm_ipc_gpu_memory_uses_configured_hw_decoders(

254 + monkeypatch: pytest.MonkeyPatch,

255 +):

256 + monkeypatch.setattr(

257 + multimodal_config_module.envs,

258 + "VLLM_VIDEO_LOADER_BACKEND",

259 + "opencv",

260 + )

261 + available_bytes = 4 * GiB_bytes

262 + mm_config = _mm_config(

263 + video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,

264 + hw_decoders=3,

265 + )

266 +

267 + assert reserve_mm_ipc_gpu_memory(available_bytes, mm_config) == (

268 + available_bytes - _pynvvideocodec_decoder_budget(hw_decoders=3)

269 + )

tests/multimodal/test_video.py

+34 -4

@@ -16,7 +16,6 @@ from transformers.video_utils import VideoMetadata

16 from vllm.assets.base import get_vllm_public_assets

17 from vllm.multimodal.video import (

18 PYNVVIDEOCODEC_DECODER_CACHE_SIZE,

19 - PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,

19 PYNVVIDEOCODEC_VIDEO_BACKEND,

20 VIDEO_LOADER_REGISTRY,

21 DynamicVideoBackend,

@@ -26,6 +25,7 @@ from vllm.multimodal.video import (

25 PyNvVideoCodecVideoBackend,

26 Qwen2VLVideoBackend,

27 Qwen3VLVideoBackend,

28 + VideoBackend,

29 VideoLoader,

30 VideoSourceMetadata,

31 VideoTargetMetadata,

@@ -199,7 +199,11 @@ def test_pynvvideocodec_codec_uses_dynamic_sampling_strategy(

199 assert metadata["frames_indices"] == [0, 9]

200

201

202 -def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatch):

202 +@pytest.mark.parametrize("hw_decoders", [1, 3])

203 +def test_pynvvideocodec_decoder_slots_are_bounded(

204 + monkeypatch: pytest.MonkeyPatch,

205 + hw_decoders: int,

206 +):

207 class FakeSlot:

208 pass

209

@@ -207,10 +211,13 @@ def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatc

211 old_slots = PyNvVideoCodecVideoBackend._decoder_slots

212 old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots

213 old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond

214 + old_max_slots = PyNvVideoCodecVideoBackend._max_decoder_slots

215 try:

216 PyNvVideoCodecVideoBackend._decoder_slots = []

217 PyNvVideoCodecVideoBackend._active_decoder_slots = 0

218 PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition()

219 + PyNvVideoCodecVideoBackend._max_decoder_slots = None

220 + PyNvVideoCodecVideoBackend._configure_decoder_slots(hw_decoders)

221

222 def fake_create_slot(cls):

223 nonlocal create_count

@@ -229,7 +236,7 @@ def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatc

236 with ExitStack() as stack:

237 retained_slots = [

238 stack.enter_context(PyNvVideoCodecVideoBackend._borrow_decoder_slot())

232 - for _ in range(PYNVVIDEOCODEC_MAX_RETAINED_DECODERS)

239 + for _ in range(hw_decoders)

240 ]

241

242 def borrow_extra_slot():

@@ -246,11 +253,34 @@ def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatc

253 assert not thread.is_alive()

254

255 assert seen_slots[0] in retained_slots

249 - assert create_count == PYNVVIDEOCODEC_MAX_RETAINED_DECODERS

256 + assert create_count == hw_decoders

257 finally:

258 PyNvVideoCodecVideoBackend._decoder_slots = old_slots

259 PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots

260 PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond

261 + PyNvVideoCodecVideoBackend._max_decoder_slots = old_max_slots

262 +

263 +

264 +def test_pynvvideocodec_decoder_slots_are_configured_once(

265 + monkeypatch: pytest.MonkeyPatch,

266 +):

267 + monkeypatch.setattr(PyNvVideoCodecVideoBackend, "_max_decoder_slots", None)

268 +

269 + PyNvVideoCodecVideoBackend._configure_decoder_slots(2)

270 + PyNvVideoCodecVideoBackend._configure_decoder_slots(2)

271 +

272 + with pytest.raises(RuntimeError, match="already configured as 2, got 3"):

273 + PyNvVideoCodecVideoBackend._configure_decoder_slots(3)

274 +

275 +

276 +@pytest.mark.parametrize("hw_decoders", [0, -1, 1.5, True, "2"])

277 +def test_pynvvideocodec_rejects_invalid_hw_decoders(hw_decoders: object):

278 + with pytest.raises(ValueError, match="hw_decoders must be a positive integer"):

279 + VideoBackend.load_bytes(

280 + b"fake video",

281 + backend=PYNVVIDEOCODEC_VIDEO_BACKEND,

282 + hw_decoders=hw_decoders, # type: ignore[arg-type]

283 + )

284

285

286 def test_pynvvideocodec_decoder_slot_retains_simple_decoder():

vllm/multimodal/gpu_ipc_memory.py

+50 -9

@@ -157,19 +157,43 @@ def reserve_mm_ipc_gpu_memory(

157 mm_config: "MultiModalConfig | None",

158 api_process_count: int = 1,

159 ) -> int:

160 - """Carve frontend multimodal GPU memory out of the KV cache.

161 -

162 - Raw decoded frames are bounded by ``mm_ipc_gpu_memory_gb`` and acquired by

163 - the frontend semaphore. Some decoders also keep persistent surfaces around;

164 - reserve a fixed upper bound for those when a GPU backend is configured.

160 + """Return KV-cache memory remaining after frontend multimodal reservations.

161 +

162 + The reservation covers:

163 +

164 + * The total ``mm_ipc_gpu_memory_gb`` budget for transient decoded-frame

165 + buffers. This budget is divided among API processes, so it is not

166 + multiplied by ``api_process_count``.

167 + * For GPU video backends, a fixed upper bound for each API process's

168 + retained decoder surfaces and CUDA context. The PyNvVideoCodec surface

169 + reservation scales with its configured ``hw_decoders`` value, and the

170 + entire decoder reservation scales with ``api_process_count`` because

171 + these resources are not shared between processes.

172 +

173 + Args:

174 + available_kv_cache_memory_bytes: KV-cache capacity before reserving

175 + memory for frontend multimodal processing.

176 + mm_config: Multimodal configuration, or ``None`` when multimodal

177 + processing is disabled.

178 + api_process_count: Number of frontend API processes sharing the GPU.

179 + Values below one are treated as one.

180 +

181 + Returns:

182 + KV-cache capacity after subtracting the frontend reservation.

183 +

184 + Raises:

185 + ValueError: If the reservation leaves no memory for the KV cache.

186 """

187 if mm_config is None:

188 return available_kv_cache_memory_bytes

189

190 + from vllm import envs

191 from vllm.multimodal.video import (

192 PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,

193 PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,

172 - PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,

194 + PYNVVIDEOCODEC_DEFAULT_HW_DECODERS,

195 + PYNVVIDEOCODEC_VIDEO_BACKEND,

196 + validate_pynvvideocodec_hw_decoders,

197 )

198

199 raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes)

@@ -177,8 +201,24 @@ def reserve_mm_ipc_gpu_memory(

201 # context on the GPU, outside the worker memory pool. Reserve that footprint

202 # per process so gpu_memory_utilization bounds total GPU usage across them.

203 num_api_servers = max(1, api_process_count)

204 + video_kwargs = mm_config.media_io_kwargs.get("video", {})

205 + video_loader_backend = (

206 + video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND

207 + )

208 + codec_backend = video_kwargs.get("backend")

209 + uses_pynvvideocodec = (

210 + video_loader_backend == PYNVVIDEOCODEC_VIDEO_BACKEND

211 + or codec_backend == PYNVVIDEOCODEC_VIDEO_BACKEND

212 + )

213 + hw_decoders = (

214 + validate_pynvvideocodec_hw_decoders(

215 + video_kwargs.get("hw_decoders", PYNVVIDEOCODEC_DEFAULT_HW_DECODERS)

216 + )

217 + if uses_pynvvideocodec

218 + else 1

219 + )

220 per_server_decoder_bytes = (

181 - PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS

221 + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * hw_decoders

222 + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES

223 )

224 decoder_reserved_bytes = (

@@ -198,8 +238,9 @@ def reserve_mm_ipc_gpu_memory(

238 f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, "

239 f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), "

240 f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is "

201 - "available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a "

202 - "different video backend, or increase gpu_memory_utilization."

241 + "available for the KV cache. Reduce mm_ipc_gpu_memory_gb or "

242 + "hw_decoders, use a different video backend, or increase "

243 + "gpu_memory_utilization."

244 )

245 logger.info_once(

246 "Reserving %s GiB of GPU memory for frontend multimodal decoding "

vllm/multimodal/media/video.py

@@ -32,6 +32,10 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):

32 runtime_kwargs: dict[str, Any] | None,

33 ) -> dict[str, Any]:

34 if runtime_kwargs:

35 + # Decoder GPU memory is reserved from the startup value.

36 + runtime_kwargs = dict(runtime_kwargs)

37 + runtime_kwargs.pop("hw_decoders", None)

38 +

39 # Block request-level selection of GPU video backends that

40 # were not configured (and VRAM-reserved) at startup.

41 for key in ("video_backend", "backend"):

vllm/multimodal/video.py

+33 -3

@@ -210,15 +210,25 @@ class VideoLoader:

210 VIDEO_LOADER_REGISTRY = VideoLoaderRegistry()

211

212 PYNVVIDEOCODEC_VIDEO_BACKEND: Literal["pynvvideocodec"] = "pynvvideocodec"

213 -# Fixed upper bound reserved for persistent PyNvVideoCodec decoder surfaces.

213 +# Per-decoder upper bound reserved for persistent PyNvVideoCodec surfaces.

214 PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES = 128 * MiB_bytes

215 PYNVVIDEOCODEC_DECODER_CACHE_SIZE = 2

216 -PYNVVIDEOCODEC_MAX_RETAINED_DECODERS = 1

216 +PYNVVIDEOCODEC_DEFAULT_HW_DECODERS = 2

217 # Per-API-server CUDA context and driver allocation, measured with

218 # PyNvVideoCodec 2.0.4 on H100.

219 PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES = int(1.8 * 1024 * MiB_bytes)

220

221

222 +def validate_pynvvideocodec_hw_decoders(hw_decoders: object) -> int:

223 + if (

224 + isinstance(hw_decoders, bool)

225 + or not isinstance(hw_decoders, int)

226 + or hw_decoders < 1

227 + ):

228 + raise ValueError("hw_decoders must be a positive integer")

229 + return hw_decoders

230 +

231 +

232 class PyNvVideoCodecDecoderSlot:

233 """A retained PyNv decoder slot and its CUDA stream.

234

@@ -628,6 +638,7 @@ class PyNvVideoCodecVideoBackendMixin:

638 _decoder_slots: ClassVar[list[PyNvVideoCodecDecoderSlot]] = []

639 _active_decoder_slots: ClassVar[int] = 0

640 _decoder_slot_cond: ClassVar[threading.Condition] = threading.Condition()

641 + _max_decoder_slots: ClassVar[int | None] = None

642 _DEVICE_INDEX: ClassVar[int] = 0

643

644 @classmethod

@@ -651,6 +662,18 @@ class PyNvVideoCodecVideoBackendMixin:

662

663 return PyNvVideoCodecDecoderSlot(torch.cuda.Stream(device=cls._DEVICE_INDEX))

664

665 + @classmethod

666 + def _configure_decoder_slots(cls, hw_decoders: object) -> None:

667 + hw_decoders = validate_pynvvideocodec_hw_decoders(hw_decoders)

668 + with cls._decoder_slot_cond:

669 + if cls._max_decoder_slots is None:

670 + cls._max_decoder_slots = hw_decoders

671 + elif cls._max_decoder_slots != hw_decoders:

672 + raise RuntimeError(

673 + "PyNvVideoCodec decoder count is already configured as "

674 + f"{cls._max_decoder_slots}, got {hw_decoders}"

675 + )

676 +

677 @staticmethod

678 @contextmanager

679 def _torch_stream_context(stream):

@@ -669,11 +692,14 @@ class PyNvVideoCodecVideoBackendMixin:

692 def _borrow_decoder_slot(cls):

693 create_slot = False

694 with cls._decoder_slot_cond:

695 + max_decoder_slots = cls._max_decoder_slots

696 + if max_decoder_slots is None:

697 + raise RuntimeError("PyNvVideoCodec decoder slots are not configured")

698 while True:

699 if cls._decoder_slots:

700 slot = cls._decoder_slots.pop()

701 break

676 - if cls._active_decoder_slots < PYNVVIDEOCODEC_MAX_RETAINED_DECODERS:

702 + if cls._active_decoder_slots < max_decoder_slots:

703 cls._active_decoder_slots += 1

704 create_slot = True

705 break

@@ -999,6 +1025,7 @@ class VideoBackend(

1025 ] = "opencv",

1026 num_ffmpeg_threads: int = 0,

1027 seek_mode: Literal["exact", "approximate"] = "exact",

1028 + hw_decoders: int = PYNVVIDEOCODEC_DEFAULT_HW_DECODERS,

1029 **kwargs,

1030 ) -> tuple[npt.NDArray, dict[str, Any]]:

1031 """Load sampled frames from raw video bytes.

@@ -1025,6 +1052,8 @@ class VideoBackend(

1052 at the cost of relying on the file's metadata. See

1053 https://meta-pytorch.org/torchcodec/stable/generated_examples/decoding/approximate_mode.html

1054 for details.

1055 + hw_decoders: Maximum number of concurrent PyNvVideoCodec decoder

1056 + slots. Defaults to 2 and must be a positive integer.

1057

1058 Returns:

1059 Tuple of ``(frames_array, metadata_dict)``.

@@ -1088,6 +1117,7 @@ class VideoBackend(

1117 "frame_recovery is not supported for "

1118 f"`{PYNVVIDEOCODEC_VIDEO_BACKEND}` backend"

1119 )

1120 + cls._configure_decoder_slots(hw_decoders)

1121 frames, source, frame_idx, valid = cls.decode_frames_pynvvideocodec(

1122 data,

1123 target,