@tduncan / Nvidia-Optimization-Stack-VLLM / commits / af16446bf3

Vram semaphore infra (#44465)

Signed-off-by: Brandon Pelfrey <bpelfrey@nvidia.com> Co-authored-by: Roger Wang <hey@rogerw.io>

Brandon Pelfrey committed Jun 26, 2026 at 17:32 UTC af16446bf39de047ab57649c933063cf1cbf1e50
11 files changed +1087 -15
requirements/cuda.txt
+1
@@ -8,6 +8,7 @@ torch==2.11.0
8 torchaudio==2.11.0
9 # These must be updated alongside torch
10 torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
11 +PyNvVideoCodec==2.0.4
12 # FlashInfer should be updated together with the Dockerfile
13 flashinfer-python==0.6.12
14 flashinfer-cubin==0.6.12
tests/multimodal/test_gpu_ipc_memory.py new
+145
@@ -0,0 +1,145 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import threading
5 +import time
6 +
7 +import pytest
8 +
9 +from vllm.multimodal.gpu_ipc_memory import (
10 + MultiModalGPUMemoryPool,
11 + get_mm_gpu_ipc_pool,
12 + maybe_init_mm_gpu_ipc_pool,
13 + set_mm_gpu_ipc_pool,
14 +)
15 +from vllm.utils.mem_constants import GiB_bytes
16 +
17 +
18 +def test_acquire_release_accounting():
19 + pool = MultiModalGPUMemoryPool(total_bytes=100)
20 + assert pool.available_bytes == 100
21 +
22 + lease = pool.acquire(40)
23 + assert pool.available_bytes == 60
24 +
25 + lease.release()
26 + assert pool.available_bytes == 100
27 +
28 +
29 +def test_acquire_too_large_raises():
30 + pool = MultiModalGPUMemoryPool(total_bytes=100)
31 + with pytest.raises(ValueError):
32 + pool.acquire(101)
33 + # Nothing should have been reserved.
34 + assert pool.available_bytes == 100
35 +
36 +
37 +def test_negative_acquire_raises():
38 + pool = MultiModalGPUMemoryPool(total_bytes=100)
39 + with pytest.raises(ValueError):
40 + pool.acquire(-1)
41 +
42 +
43 +def test_double_release_is_noop():
44 + pool = MultiModalGPUMemoryPool(total_bytes=100)
45 + lease = pool.acquire(50)
46 + lease.release()
47 + assert pool.available_bytes == 100
48 + # Releasing again must not inflate the pool past its capacity.
49 + lease.release()
50 + assert pool.available_bytes == 100
51 +
52 +
53 +def test_context_manager_releases_on_exception():
54 + pool = MultiModalGPUMemoryPool(total_bytes=100)
55 + with pytest.raises(RuntimeError), pool.acquire(50):
56 + assert pool.available_bytes == 50
57 + raise RuntimeError("boom")
58 + assert pool.available_bytes == 100
59 +
60 +
61 +def test_acquire_blocks_until_release():
62 + pool = MultiModalGPUMemoryPool(total_bytes=100)
63 + first = pool.acquire(80)
64 +
65 + acquired = threading.Event()
66 +
67 + def waiter():
68 + # Needs 50 bytes but only 20 are free; must block until `first`
69 + # is released.
70 + with pool.acquire(50):
71 + acquired.set()
72 +
73 + t = threading.Thread(target=waiter)
74 + t.start()
75 +
76 + # The waiter cannot proceed yet.
77 + assert not acquired.wait(timeout=0.2)
78 +
79 + # Releasing the first lease frees enough budget to unblock the waiter.
80 + first.release()
81 + assert acquired.wait(timeout=2.0)
82 + t.join(timeout=2.0)
83 + assert not t.is_alive()
84 + assert pool.available_bytes == 100
85 +
86 +
87 +def test_concurrent_acquires_serialize():
88 + pool = MultiModalGPUMemoryPool(total_bytes=100)
89 + # Each task needs 60 bytes, so only one can hold the budget at a time.
90 + in_section = []
91 + max_concurrent = 0
92 + lock = threading.Lock()
93 +
94 + def task():
95 + nonlocal max_concurrent
96 + with pool.acquire(60):
97 + with lock:
98 + in_section.append(1)
99 + max_concurrent = max(max_concurrent, len(in_section))
100 + time.sleep(0.05)
101 + with lock:
102 + in_section.pop()
103 +
104 + threads = [threading.Thread(target=task) for _ in range(4)]
105 + for t in threads:
106 + t.start()
107 + for t in threads:
108 + t.join(timeout=5.0)
109 + assert not t.is_alive()
110 +
111 + assert max_concurrent == 1
112 + assert pool.available_bytes == 100
113 +
114 +
115 +def test_zero_total_bytes_rejected():
116 + with pytest.raises(ValueError):
117 + MultiModalGPUMemoryPool(total_bytes=0)
118 +
119 +
120 +def test_global_pool_accessor():
121 + try:
122 + assert maybe_init_mm_gpu_ipc_pool(0) is None
123 + assert get_mm_gpu_ipc_pool() is None
124 +
125 + pool = maybe_init_mm_gpu_ipc_pool(2)
126 + assert pool is not None
127 + assert get_mm_gpu_ipc_pool() is pool
128 + assert pool.total_bytes == 2 * GiB_bytes
129 + finally:
130 + set_mm_gpu_ipc_pool(None)
131 +
132 +
133 +def test_global_pool_splits_budget_across_api_processes():
134 + try:
135 + pool = maybe_init_mm_gpu_ipc_pool(2, api_process_count=4)
136 + assert pool is not None
137 + assert get_mm_gpu_ipc_pool() is pool
138 + assert pool.total_bytes == GiB_bytes // 2
139 + finally:
140 + set_mm_gpu_ipc_pool(None)
141 +
142 +
143 +def test_global_pool_rejects_invalid_api_process_count():
144 + with pytest.raises(ValueError):
145 + maybe_init_mm_gpu_ipc_pool(2, api_process_count=0)
tests/multimodal/test_video.py
+234
@@ -1,6 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 import itertools
5 +import sys
6 +import threading
7 +from contextlib import ExitStack, contextmanager
8 from pathlib import Path
9
10 import numpy as np
@@ -11,10 +15,15 @@ from transformers.video_utils import VideoMetadata
15
16 from vllm.assets.base import get_vllm_public_assets
17 from vllm.multimodal.video import (
18 + PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
19 + PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
20 + PYNVVIDEOCODEC_VIDEO_BACKEND,
21 VIDEO_LOADER_REGISTRY,
22 DynamicVideoBackend,
23 GLM46VVideoBackend,
24 Molmo2VideoBackend,
25 + PyNvVideoCodecDecoderSlot,
26 + PyNvVideoCodecVideoBackend,
27 Qwen2VLVideoBackend,
28 Qwen3VLVideoBackend,
29 VideoLoader,
@@ -22,6 +31,7 @@ from vllm.multimodal.video import (
31 VideoTargetMetadata,
32 get_video_loader_backend_for_processor,
33 )
34 +from vllm.platforms import current_platform
35 from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
36
37 from .utils import create_long_gop_video, create_video_from_image
@@ -65,6 +75,230 @@ def test_video_loader_type_doesnt_exist():
75 VIDEO_LOADER_REGISTRY.load("non_existing_video_loader")
76
77
78 +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
79 +def test_pynvvideocodec_backend_accounts_raw_decoded_frames(
80 + monkeypatch: pytest.MonkeyPatch,
81 +):
82 + decoder_cache_sizes = []
83 +
84 + class FakeMetadata:
85 + width = 10
86 + height = 20
87 + average_fps = 5.0
88 + duration = 2.0
89 +
90 + class FakeDecoder:
91 + def __init__(self, *args, **kwargs):
92 + decoder_cache_sizes.append(kwargs["decoder_cache_size"])
93 +
94 + def __len__(self):
95 + return 10
96 +
97 + def get_stream_metadata(self):
98 + return FakeMetadata()
99 +
100 + class FakeNvc:
101 + class OutputColorType:
102 + RGB = "rgb"
103 +
104 + SimpleDecoder = FakeDecoder
105 +
106 + class RecordingPool:
107 + def __init__(self):
108 + self.acquired: list[int] = []
109 +
110 + @contextmanager
111 + def acquire(self, size: int):
112 + self.acquired.append(size)
113 + yield
114 +
115 + def fake_decode(cls, file_path: str, frame_idx: list[int], nvc):
116 + return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8)
117 +
118 + pool = RecordingPool()
119 + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc)
120 + monkeypatch.setattr(
121 + "vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool
122 + )
123 + monkeypatch.setattr(
124 + PyNvVideoCodecVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode)
125 + )
126 +
127 + loader = VIDEO_LOADER_REGISTRY.load(PYNVVIDEOCODEC_VIDEO_BACKEND)
128 + frames, metadata = loader.load_bytes(b"fake video", num_frames=4)
129 +
130 + assert frames.shape == (4, 20, 10, 3)
131 + assert pool.acquired == [4 * 20 * 10 * 3]
132 + assert decoder_cache_sizes == [PYNVVIDEOCODEC_DECODER_CACHE_SIZE]
133 + assert metadata["video_backend"] == PYNVVIDEOCODEC_VIDEO_BACKEND
134 + assert metadata["frames_indices"] == [0, 3, 6, 9]
135 +
136 +
137 +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
138 +def test_pynvvideocodec_codec_uses_dynamic_sampling_strategy(
139 + monkeypatch: pytest.MonkeyPatch,
140 +):
141 + decoded_indices = []
142 +
143 + class FakeMetadata:
144 + width = 10
145 + height = 20
146 + average_fps = 5.0
147 + duration = 2.0
148 +
149 + class FakeDecoder:
150 + def __init__(self, *args, **kwargs):
151 + pass
152 +
153 + def __len__(self):
154 + return 10
155 +
156 + def get_stream_metadata(self):
157 + return FakeMetadata()
158 +
159 + class FakeNvc:
160 + class OutputColorType:
161 + RGB = "rgb"
162 +
163 + SimpleDecoder = FakeDecoder
164 +
165 + class RecordingPool:
166 + def __init__(self):
167 + self.acquired: list[int] = []
168 +
169 + @contextmanager
170 + def acquire(self, size: int):
171 + self.acquired.append(size)
172 + yield
173 +
174 + def fake_decode(cls, file_path: str, frame_idx: list[int], nvc):
175 + decoded_indices.append(frame_idx)
176 + return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8)
177 +
178 + pool = RecordingPool()
179 + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc)
180 + monkeypatch.setattr(
181 + "vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool
182 + )
183 + monkeypatch.setattr(
184 + DynamicVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode)
185 + )
186 +
187 + loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
188 + frames, metadata = loader.load_bytes(
189 + b"fake video",
190 + fps=2,
191 + max_duration=1,
192 + backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
193 + )
194 +
195 + assert frames.shape == (2, 20, 10, 3)
196 + assert decoded_indices == [[0, 9]]
197 + assert pool.acquired == [2 * 20 * 10 * 3]
198 + assert metadata["video_backend"] == f"{PYNVVIDEOCODEC_VIDEO_BACKEND}_dynamic"
199 + assert metadata["frames_indices"] == [0, 9]
200 +
201 +
202 +def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatch):
203 + class FakeSlot:
204 + pass
205 +
206 + create_count = 0
207 + old_slots = PyNvVideoCodecVideoBackend._decoder_slots
208 + old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots
209 + old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond
210 + try:
211 + PyNvVideoCodecVideoBackend._decoder_slots = []
212 + PyNvVideoCodecVideoBackend._active_decoder_slots = 0
213 + PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition()
214 +
215 + def fake_create_slot(cls):
216 + nonlocal create_count
217 + create_count += 1
218 + return FakeSlot()
219 +
220 + monkeypatch.setattr(
221 + PyNvVideoCodecVideoBackend,
222 + "_create_decoder_slot",
223 + classmethod(fake_create_slot),
224 + )
225 +
226 + borrowed = threading.Event()
227 + seen_slots = []
228 +
229 + with ExitStack() as stack:
230 + retained_slots = [
231 + stack.enter_context(PyNvVideoCodecVideoBackend._borrow_decoder_slot())
232 + for _ in range(PYNVVIDEOCODEC_MAX_RETAINED_DECODERS)
233 + ]
234 +
235 + def borrow_extra_slot():
236 + with PyNvVideoCodecVideoBackend._borrow_decoder_slot() as extra_slot:
237 + seen_slots.append(extra_slot)
238 + borrowed.set()
239 +
240 + thread = threading.Thread(target=borrow_extra_slot)
241 + thread.start()
242 + assert not borrowed.wait(timeout=0.2)
243 +
244 + assert borrowed.wait(timeout=2.0)
245 + thread.join(timeout=2.0)
246 + assert not thread.is_alive()
247 +
248 + assert seen_slots[0] in retained_slots
249 + assert create_count == PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
250 + finally:
251 + PyNvVideoCodecVideoBackend._decoder_slots = old_slots
252 + PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots
253 + PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond
254 +
255 +
256 +def test_pynvvideocodec_decoder_slot_retains_simple_decoder():
257 + events: list[tuple[object, ...]] = []
258 +
259 + class FakeStream:
260 + cuda_stream = "cuda-stream"
261 +
262 + class FakeDecoder:
263 + def __init__(self, file_path: str, **kwargs):
264 + events.append(
265 + (
266 + "create",
267 + file_path,
268 + kwargs["gpu_id"],
269 + kwargs["cuda_stream"],
270 + kwargs["decoder_cache_size"],
271 + )
272 + )
273 +
274 + def reconfigure_decoder(self, file_path: str):
275 + events.append(("reconfigure", file_path))
276 +
277 + class FakeNvc:
278 + class OutputColorType:
279 + RGB = "rgb"
280 +
281 + SimpleDecoder = FakeDecoder
282 +
283 + slot = PyNvVideoCodecDecoderSlot(FakeStream())
284 +
285 + decoder = slot.get_decoder("first.mp4", FakeNvc, device_index=7)
286 + assert slot.get_decoder("first.mp4", FakeNvc, device_index=7) is decoder
287 + assert slot.get_decoder("second.mp4", FakeNvc, device_index=7) is decoder
288 +
289 + assert events == [
290 + (
291 + "create",
292 + "first.mp4",
293 + 7,
294 + "cuda-stream",
295 + PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
296 + ),
297 + ("reconfigure", "second.mp4"),
298 + ]
299 + assert slot.source_path == "second.mp4"
300 +
301 +
302 # ============================================================================
303 # Video Processor → Video Loader Tests (via model repo)
304 # ============================================================================
tests/v1/worker/test_gpu_worker.py new
+116
@@ -0,0 +1,116 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from types import SimpleNamespace
5 +
6 +import pytest
7 +
8 +import vllm.v1.worker.gpu_worker as gpu_worker_module
9 +from vllm.multimodal.video import (
10 + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
11 + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
12 + PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
13 + PYNVVIDEOCODEC_VIDEO_BACKEND,
14 +)
15 +from vllm.utils.mem_constants import GiB_bytes
16 +from vllm.v1.worker.gpu_worker import Worker
17 +
18 +
19 +def _worker_with_mm_config(
20 + mm_config: SimpleNamespace,
21 + *,
22 + api_process_count: int = 1,
23 +) -> Worker:
24 + worker = object.__new__(Worker)
25 + worker.model_config = SimpleNamespace(multimodal_config=mm_config)
26 + worker.parallel_config = SimpleNamespace(_api_process_count=api_process_count)
27 + return worker
28 +
29 +
30 +def _mm_config(
31 + *,
32 + mm_ipc_gpu_memory_gb: float = 0,
33 + video_backend: str | None = None,
34 +) -> SimpleNamespace:
35 + video_kwargs = {} if video_backend is None else {"video_backend": video_backend}
36 + return SimpleNamespace(
37 + mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
38 + media_io_kwargs={"video": video_kwargs} if video_kwargs else {},
39 + )
40 +
41 +
42 +def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:
43 + return api_process_count * (
44 + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
45 + + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
46 + )
47 +
48 +
49 +@pytest.mark.parametrize("video_backend", [None, "opencv"])
50 +def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
51 + monkeypatch: pytest.MonkeyPatch,
52 + video_backend: str | None,
53 +):
54 + monkeypatch.setattr(
55 + gpu_worker_module.envs,
56 + "VLLM_VIDEO_LOADER_BACKEND",
57 + "opencv",
58 + )
59 + worker = _worker_with_mm_config(
60 + _mm_config(mm_ipc_gpu_memory_gb=0.25, video_backend=video_backend)
61 + )
62 +
63 + assert worker._reserve_mm_ipc_gpu_memory(GiB_bytes) == int(0.75 * GiB_bytes)
64 +
65 +
66 +def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
67 + monkeypatch: pytest.MonkeyPatch,
68 +):
69 + monkeypatch.setattr(
70 + gpu_worker_module.envs,
71 + "VLLM_VIDEO_LOADER_BACKEND",
72 + "opencv",
73 + )
74 + worker = _worker_with_mm_config(
75 + _mm_config(
76 + mm_ipc_gpu_memory_gb=0.25,
77 + video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
78 + )
79 + )
80 + available_bytes = 4 * GiB_bytes
81 +
82 + assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
83 + available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget()
84 + )
85 +
86 +
87 +def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
88 + monkeypatch: pytest.MonkeyPatch,
89 +):
90 + monkeypatch.setattr(
91 + gpu_worker_module.envs,
92 + "VLLM_VIDEO_LOADER_BACKEND",
93 + PYNVVIDEOCODEC_VIDEO_BACKEND,
94 + )
95 + worker = _worker_with_mm_config(_mm_config())
96 + available_bytes = 4 * GiB_bytes
97 +
98 + assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
99 + available_bytes - _pynvvideocodec_decoder_budget()
100 + )
101 +
102 +
103 +def test_reserve_mm_ipc_gpu_memory_scales_pynvvideocodec_budget_by_api_servers(
104 + monkeypatch: pytest.MonkeyPatch,
105 +):
106 + monkeypatch.setattr(
107 + gpu_worker_module.envs,
108 + "VLLM_VIDEO_LOADER_BACKEND",
109 + PYNVVIDEOCODEC_VIDEO_BACKEND,
110 + )
111 + worker = _worker_with_mm_config(_mm_config(), api_process_count=3)
112 + available_bytes = 8 * GiB_bytes
113 +
114 + assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
115 + available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)
116 + )
vllm/config/model.py
+4
@@ -351,6 +351,7 @@ class ModelConfig:
351 skip_mm_profiling: InitVar[bool | None] = None
352 video_pruning_rate: InitVar[float | None] = None
353 mm_tensor_ipc: InitVar[MMTensorIPC] = None
354 + mm_ipc_gpu_memory_gb: InitVar[float | None] = None
355
356 def compute_hash(self) -> str:
357 """
@@ -397,6 +398,7 @@ class ModelConfig:
398 "mm_encoder_tp_mode",
399 "interleave_mm_strings",
400 "skip_mm_profiling",
401 + "mm_ipc_gpu_memory_gb",
402 }
403
404 from vllm.config.utils import get_hash_factors, hash_factors
@@ -477,6 +479,7 @@ class ModelConfig:
479 skip_mm_profiling: bool | None,
480 video_pruning_rate: float | None,
481 mm_tensor_ipc: MMTensorIPC,
482 + mm_ipc_gpu_memory_gb: float | None,
483 ) -> None:
484 # Keep set served_model_name before maybe_model_redirect(self.model)
485 self.served_model_name = get_served_model_name(
@@ -690,6 +693,7 @@ class ModelConfig:
693 skip_mm_profiling=skip_mm_profiling,
694 video_pruning_rate=video_pruning_rate,
695 mm_tensor_ipc=mm_tensor_ipc,
696 + mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
697 )
698
699 mm_config_kwargs = {
vllm/config/multimodal.py
+10
@@ -197,6 +197,16 @@ class MultiModalConfig:
197 - "direct_rpc": Use msgspec serialization via RPC
198 - "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC
199 Defaults to "direct_rpc". """
200 + mm_ipc_gpu_memory_gb: float = Field(default=0, ge=0)
201 + """Amount of GPU memory (in GiB) sequestered on the engine's device for
202 + GPU-side multimodal work in the API-server (frontend) process, such as
203 + hardware video decoding.
204 +
205 + This budget is carved out of the engine's KV-cache memory so the headroom
206 + physically exists, and frontend GPU decode paths acquire from a blocking
207 + byte-counting semaphore of this size before allocating on the device.
208 +
209 + Set to `0` (default) to disable frontend GPU multimodal memory gating."""
210
211 @field_validator("limit_per_prompt", mode="before")
212 @classmethod
vllm/engine/arg_utils.py
+6
@@ -582,6 +582,7 @@ class EngineArgs:
582 skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
583 video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate
584 mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc
585 + mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb
586 # LoRA fields
587 enable_lora: bool = False
588 max_loras: int = LoRAConfig.max_loras
@@ -1294,6 +1295,10 @@ class EngineArgs:
1295 multimodal_group.add_argument(
1296 "--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"]
1297 )
1298 + multimodal_group.add_argument(
1299 + "--mm-ipc-gpu-memory-gb",
1300 + **multimodal_kwargs["mm_ipc_gpu_memory_gb"],
1301 + )
1302
1303 # LoRA related configs
1304 lora_kwargs = get_kwargs(LoRAConfig)
@@ -1660,6 +1665,7 @@ class EngineArgs:
1665 logits_processors=self.logits_processors,
1666 video_pruning_rate=self.video_pruning_rate,
1667 mm_tensor_ipc=self.mm_tensor_ipc,
1668 + mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb,
1669 io_processor_plugin=self.io_processor_plugin,
1670 renderer_num_workers=self.renderer_num_workers,
1671 )
vllm/multimodal/gpu_ipc_memory.py new
+147
@@ -0,0 +1,147 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Admission control for frontend GPU-side multimodal work.
4 +
5 +When multimodal media is decoded on the GPU in the API-server (frontend)
6 +process, the decoded buffers compete for the same device memory that the
7 +engine reserves for weights, activations, and the KV cache. To keep the
8 +frontend's GPU usage within a sequestered budget (see
9 +``MultiModalConfig.mm_ipc_gpu_memory_gb``), decode paths acquire the number of
10 +bytes they need from a process-global :class:`MultiModalGPUMemoryPool` before
11 +allocating on the device and release them once the device memory is freed.
12 +
13 +The pool is a simple byte-counting semaphore: ``acquire`` blocks until enough
14 +budget is free, so concurrent requests serialize rather than oversubscribe the
15 +GPU. It lives only in the frontend process; the engine carves the matching
16 +amount out of its KV-cache budget so the headroom physically exists.
17 +"""
18 +
19 +import threading
20 +
21 +from vllm.logger import init_logger
22 +from vllm.utils.mem_constants import GiB_bytes
23 +
24 +logger = init_logger(__name__)
25 +
26 +
27 +class MultiModalGPUMemoryLease:
28 + """A handle for bytes acquired from a :class:`MultiModalGPUMemoryPool`.
29 +
30 + Releasing is idempotent and the lease doubles as a context manager so the
31 + budget is returned even if the decode raises.
32 + """
33 +
34 + def __init__(self, pool: "MultiModalGPUMemoryPool", lease_id: int, nbytes: int):
35 + self.lease_id = lease_id
36 + self.nbytes = nbytes
37 + self._pool = pool
38 +
39 + def release(self) -> None:
40 + self._pool._release(self)
41 +
42 + def __enter__(self) -> "MultiModalGPUMemoryLease":
43 + return self
44 +
45 + def __exit__(self, *exc_info) -> None:
46 + self.release()
47 +
48 +
49 +class MultiModalGPUMemoryPool:
50 + """Blocking byte-counting semaphore for frontend GPU multimodal memory.
51 +
52 + Thread-safe in both directions: ``acquire`` (blocking) and ``release`` are
53 + typically called from the renderer's multimodal executor threads.
54 + """
55 +
56 + def __init__(self, total_bytes: int):
57 + if total_bytes <= 0:
58 + raise ValueError(f"total_bytes must be positive, got {total_bytes}")
59 + self._total_bytes = total_bytes
60 + self._available = total_bytes
61 + self._cond = threading.Condition()
62 + self._next_lease_id = 0
63 + # Outstanding lease ids, so a double release is a no-op.
64 + self._outstanding: set[int] = set()
65 +
66 + @property
67 + def total_bytes(self) -> int:
68 + return self._total_bytes
69 +
70 + @property
71 + def available_bytes(self) -> int:
72 + with self._cond:
73 + return self._available
74 +
75 + def acquire(self, nbytes: int) -> MultiModalGPUMemoryLease:
76 + """Reserve ``nbytes``, blocking until that much budget is free.
77 +
78 + Raises ``ValueError`` if ``nbytes`` exceeds the pool's total capacity,
79 + since such a request could never be satisfied.
80 + """
81 + if nbytes < 0:
82 + raise ValueError(f"Cannot acquire negative bytes: {nbytes}")
83 + if nbytes > self._total_bytes:
84 + raise ValueError(
85 + f"Multimodal GPU decode requested {nbytes} bytes, which exceeds "
86 + f"the total pool size of {self._total_bytes} bytes. Increase "
87 + f"--mm-ipc-gpu-memory-gb or reduce the multimodal input size."
88 + )
89 + with self._cond:
90 + while self._available < nbytes:
91 + self._cond.wait()
92 + self._available -= nbytes
93 + lease_id = self._next_lease_id
94 + self._next_lease_id += 1
95 + self._outstanding.add(lease_id)
96 + return MultiModalGPUMemoryLease(self, lease_id, nbytes)
97 +
98 + def _release(self, lease: MultiModalGPUMemoryLease) -> None:
99 + with self._cond:
100 + if lease.lease_id not in self._outstanding:
101 + # Already released — idempotent.
102 + return
103 + self._outstanding.discard(lease.lease_id)
104 + self._available += lease.nbytes
105 + self._cond.notify_all()
106 +
107 +
108 +_GLOBAL_POOL: MultiModalGPUMemoryPool | None = None
109 +
110 +
111 +def set_mm_gpu_ipc_pool(pool: MultiModalGPUMemoryPool | None) -> None:
112 + """Install the process-global pool (frontend process only)."""
113 + global _GLOBAL_POOL
114 + _GLOBAL_POOL = pool
115 +
116 +
117 +def get_mm_gpu_ipc_pool() -> MultiModalGPUMemoryPool | None:
118 + """Return the process-global pool, or ``None`` when gating is disabled."""
119 + return _GLOBAL_POOL
120 +
121 +
122 +def maybe_init_mm_gpu_ipc_pool(
123 + mm_ipc_gpu_memory_gb: float,
124 + api_process_count: int = 1,
125 +) -> MultiModalGPUMemoryPool | None:
126 + """Create and install the global pool from the configured GiB budget.
127 +
128 + Returns ``None`` (and leaves gating disabled) when the budget is 0. When
129 + multiple API-server processes share one engine, each process gets an equal
130 + slice of the user-provided frontend budget.
131 + """
132 + if mm_ipc_gpu_memory_gb <= 0:
133 + set_mm_gpu_ipc_pool(None)
134 + return None
135 + if api_process_count <= 0:
136 + raise ValueError(f"api_process_count must be positive, got {api_process_count}")
137 + total_bytes = int(mm_ipc_gpu_memory_gb * GiB_bytes) // api_process_count
138 + pool = MultiModalGPUMemoryPool(total_bytes)
139 + set_mm_gpu_ipc_pool(pool)
140 + logger.info(
141 + "Initialized multimodal GPU IPC memory pool with %d bytes for this API "
142 + "process (%.2f GiB total budget across %d API process(es)).",
143 + total_bytes,
144 + mm_ipc_gpu_memory_gb,
145 + api_process_count,
146 + )
147 + return pool
vllm/multimodal/video.py
+331 -13
@@ -1,7 +1,11 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 import math
4 +import os
5 +import tempfile
6 +import threading
7 from abc import abstractmethod
8 +from contextlib import contextmanager, suppress
9 from io import BytesIO
10 from typing import Any, ClassVar, Literal, NamedTuple, cast
11
@@ -11,6 +15,7 @@ import torch
15
16 from vllm.logger import init_logger
17 from vllm.utils.import_utils import PlaceholderModule
18 +from vllm.utils.mem_constants import MiB_bytes
19 from vllm.utils.registry import ExtensionManager
20
21 try:
@@ -130,6 +135,14 @@ class VideoSourceMetadata(NamedTuple):
135 duration: float
136
137
138 +class PyNvVideoCodecSourceMetadata(NamedTuple):
139 + """Metadata needed before GPU video decode."""
140 +
141 + source: VideoSourceMetadata
142 + width: int
143 + height: int
144 +
145 +
146 class VideoLoader:
147 @classmethod
148 def compute_frames_index_to_sample(
@@ -170,6 +183,57 @@ class VideoLoader:
183
184 VIDEO_LOADER_REGISTRY = VideoLoaderRegistry()
185
186 +PYNVVIDEOCODEC_VIDEO_BACKEND: Literal["pynvvideocodec"] = "pynvvideocodec"
187 +# Fixed upper bound reserved for persistent PyNvVideoCodec decoder surfaces.
188 +PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES = 128 * MiB_bytes
189 +PYNVVIDEOCODEC_DECODER_CACHE_SIZE = 2
190 +PYNVVIDEOCODEC_MAX_RETAINED_DECODERS = 1
191 +# Per-API-server CUDA context and driver allocation, measured with
192 +# PyNvVideoCodec 2.0.4 on H100.
193 +PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES = int(1.8 * 1024 * MiB_bytes)
194 +
195 +
196 +class PyNvVideoCodecDecoderSlot:
197 + """A retained PyNv decoder slot and its CUDA stream.
198 +
199 + The decoder is reused across requests: ``reconfigure_decoder`` repoints the
200 + existing decoder at each new source instead of paying a fresh
201 + ``SimpleDecoder`` construction per request. Construction (CUVID parser +
202 + decoder + surface-pool allocation) is the dominant per-request cost, so
203 + reconfiguring is far cheaper. A single decoder serves both metadata
204 + (``len``/``get_stream_metadata``) and frame decode -- no separate
205 + metadata decoder.
206 + """
207 +
208 + def __init__(self, stream) -> None:
209 + self.stream = stream
210 + self.decoder = None
211 + self.source_path: str | None = None
212 +
213 + def _construct(self, file_path: str, nvc, device_index: int) -> None:
214 + self.decoder = nvc.SimpleDecoder(
215 + file_path,
216 + output_color_type=nvc.OutputColorType.RGB,
217 + use_device_memory=True,
218 + need_scanned_stream_metadata=True,
219 + gpu_id=device_index,
220 + cuda_stream=self.stream.cuda_stream,
221 + decoder_cache_size=PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
222 + )
223 + self.source_path = file_path
224 +
225 + def get_decoder(self, file_path: str, nvc, device_index: int):
226 + if self.decoder is None:
227 + self._construct(file_path, nvc, device_index)
228 + elif self.source_path != file_path:
229 + try:
230 + self.decoder.reconfigure_decoder(file_path)
231 + self.source_path = file_path
232 + except Exception:
233 + # reconfigure unsupported/unsafe for this source -> rebuild.
234 + self._construct(file_path, nvc, device_index)
235 + return self.decoder
236 +
237
238 class OpenCVVideoBackendMixin:
239 @staticmethod
@@ -485,15 +549,223 @@ class PyAVVideoBackendMixin:
549 return np.stack(frames_list), valid_indices
550
551
552 +class PyNvVideoCodecVideoBackendMixin:
553 + """PyNvVideoCodec utilities for GPU-backed frame decode."""
554 +
555 + _decoder_slots: ClassVar[list[PyNvVideoCodecDecoderSlot]] = []
556 + _active_decoder_slots: ClassVar[int] = 0
557 + _decoder_slot_cond: ClassVar[threading.Condition] = threading.Condition()
558 + _DEVICE_INDEX: ClassVar[int] = 0
559 +
560 + @classmethod
561 + @abstractmethod
562 + def compute_frames_index_to_sample(
563 + cls,
564 + source: VideoSourceMetadata,
565 + target: VideoTargetMetadata,
566 + **kwargs,
567 + ) -> list[int]:
568 + raise NotImplementedError
569 +
570 + @classmethod
571 + @abstractmethod
572 + def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:
573 + raise NotImplementedError
574 +
575 + @classmethod
576 + def _create_decoder_slot(cls) -> PyNvVideoCodecDecoderSlot:
577 + import torch
578 +
579 + return PyNvVideoCodecDecoderSlot(torch.cuda.Stream(device=cls._DEVICE_INDEX))
580 +
581 + @staticmethod
582 + @contextmanager
583 + def _torch_stream_context(stream):
584 + import torch
585 +
586 + torch.accelerator.set_device_index(stream.device.index)
587 + previous_stream = torch.accelerator.current_stream()
588 + torch.accelerator.set_stream(stream)
589 + try:
590 + yield
591 + finally:
592 + torch.accelerator.set_stream(previous_stream)
593 +
594 + @classmethod
595 + @contextmanager
596 + def _borrow_decoder_slot(cls):
597 + create_slot = False
598 + with cls._decoder_slot_cond:
599 + while True:
600 + if cls._decoder_slots:
601 + slot = cls._decoder_slots.pop()
602 + break
603 + if cls._active_decoder_slots < PYNVVIDEOCODEC_MAX_RETAINED_DECODERS:
604 + cls._active_decoder_slots += 1
605 + create_slot = True
606 + break
607 + cls._decoder_slot_cond.wait()
608 +
609 + if create_slot:
610 + try:
611 + slot = cls._create_decoder_slot()
612 + except Exception:
613 + with cls._decoder_slot_cond:
614 + cls._active_decoder_slots -= 1
615 + cls._decoder_slot_cond.notify()
616 + raise
617 +
618 + try:
619 + yield slot
620 + finally:
621 + with cls._decoder_slot_cond:
622 + cls._decoder_slots.append(slot)
623 + cls._decoder_slot_cond.notify()
624 +
625 + @staticmethod
626 + def _metadata_value(metadata, *names: str, default=None):
627 + for name in names:
628 + value = getattr(metadata, name, None)
629 + if value is not None:
630 + return value
631 + return default
632 +
633 + @classmethod
634 + def _read_source_metadata(
635 + cls,
636 + file_path: str,
637 + nvc,
638 + ) -> PyNvVideoCodecSourceMetadata:
639 + with cls._borrow_decoder_slot() as decoder_slot:
640 + with cls._torch_stream_context(decoder_slot.stream):
641 + decoder = decoder_slot.get_decoder(
642 + file_path, nvc, device_index=cls._DEVICE_INDEX
643 + )
644 + metadata = decoder.get_stream_metadata()
645 + total_frames_num = len(decoder)
646 + width = int(cls._metadata_value(metadata, "width", default=0))
647 + height = int(cls._metadata_value(metadata, "height", default=0))
648 + original_fps = float(
649 + cls._metadata_value(
650 + metadata,
651 + "average_fps",
652 + "avg_frame_rate",
653 + "frame_rate",
654 + "frameRate",
655 + default=0.0,
656 + )
657 + )
658 + duration = float(
659 + cls._metadata_value(metadata, "duration", default=0.0)
660 + or (total_frames_num / original_fps if original_fps > 0 else 0.0)
661 + )
662 + if total_frames_num <= 0:
663 + raise ValueError("Could not determine video frame count")
664 + if width <= 0 or height <= 0:
665 + raise ValueError("Could not determine video dimensions")
666 + return PyNvVideoCodecSourceMetadata(
667 + source=VideoSourceMetadata(total_frames_num, original_fps, duration),
668 + width=width,
669 + height=height,
670 + )
671 +
672 + @classmethod
673 + def _decode_to_pinned_host(
674 + cls,
675 + file_path: str,
676 + frame_idx: list[int],
677 + nvc,
678 + ) -> npt.NDArray:
679 + import torch
680 +
681 + if not frame_idx:
682 + return np.empty((0,), dtype=np.uint8)
683 +
684 + with cls._borrow_decoder_slot() as decoder_slot:
685 + stream = decoder_slot.stream
686 + with cls._torch_stream_context(stream):
687 + decoder = decoder_slot.get_decoder(
688 + file_path, nvc, device_index=cls._DEVICE_INDEX
689 + )
690 + decoded_frames = decoder.get_batch_frames_by_index(frame_idx)
691 + if len(decoded_frames) < len(frame_idx):
692 + logger.warning(
693 + "pynvvideocodec video loading: expected %d frames but got %d.",
694 + len(frame_idx),
695 + len(decoded_frames),
696 + )
697 + torch_frames = [torch.from_dlpack(frame) for frame in decoded_frames]
698 + if not torch_frames:
699 + return np.empty((0,), dtype=np.uint8)
700 + device_frames = torch.stack(torch_frames)
701 + if device_frames.ndim != 4:
702 + raise ValueError(
703 + "PyNvVideoCodec returned frames with unexpected shape "
704 + f"{tuple(device_frames.shape)}"
705 + )
706 + device_frames = device_frames.permute(0, 3, 1, 2).contiguous()
707 + host_frames = torch.empty(
708 + device_frames.shape,
709 + dtype=device_frames.dtype,
710 + device="cpu",
711 + pin_memory=True,
712 + )
713 + host_frames.copy_(device_frames, non_blocking=True)
714 + stream.synchronize()
715 + host_array = host_frames.numpy()
716 + del decoded_frames, torch_frames, device_frames
717 + return host_array
718 +
719 + @classmethod
720 + def decode_frames_pynvvideocodec(
721 + cls,
722 + data: bytes,
723 + target: VideoTargetMetadata,
724 + **kwargs,
725 + ) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]:
726 + import PyNvVideoCodec as nvc
727 +
728 + from vllm.multimodal.gpu_ipc_memory import get_mm_gpu_ipc_pool
729 +
730 + temp_fd, temp_path = tempfile.mkstemp(suffix=".mp4")
731 + try:
732 + with os.fdopen(temp_fd, "wb") as temp_file:
733 + temp_file.write(data)
734 +
735 + gpu_source = cls._read_source_metadata(temp_path, nvc)
736 + source = cls._prepare_source(gpu_source.source)
737 + frame_idx = cls.compute_frames_index_to_sample(
738 + source=source, target=target, **kwargs
739 + )
740 + raw_frame_bytes = len(frame_idx) * gpu_source.height * gpu_source.width * 3
741 + pool = get_mm_gpu_ipc_pool()
742 + if pool is None or raw_frame_bytes == 0:
743 + frames = cls._decode_to_pinned_host(temp_path, frame_idx, nvc)
744 + else:
745 + with pool.acquire(raw_frame_bytes):
746 + frames = cls._decode_to_pinned_host(temp_path, frame_idx, nvc)
747 + finally:
748 + with suppress(FileNotFoundError):
749 + os.unlink(temp_path)
750 +
751 + valid_frame_indices = frame_idx[: int(frames.shape[0])]
752 + return frames, source, frame_idx, valid_frame_indices
753 +
754 +
755 @VIDEO_LOADER_REGISTRY.register("opencv")
489 -class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
756 +class VideoBackend(
757 + VideoLoader,
758 + OpenCVVideoBackendMixin,
759 + PyAVVideoBackendMixin,
760 + PyNvVideoCodecVideoBackendMixin,
761 +):
762 """Uniform-sampling video backend.
763
764 Samples ``num_frames`` uniformly across the video (or one frame every
765 ``1/fps`` seconds, whichever produces fewer frames). The decoding codec
494 - is selected via the ``backend`` kwarg (``"opencv"`` or ``"pyav"``),
495 - which can be passed through ``--media-io-kwargs``. Defaults to
496 - ``"pyav"`` for concurrent decoding.
766 + is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``, or
767 + ``"pynvvideocodec"``), which can be passed through
768 + ``--media-io-kwargs``. Defaults to ``"opencv"``.
769 """
770
771 _sampling_suffix: ClassVar[str] = ""
@@ -538,7 +810,7 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
810 max_duration: int = 300,
811 frame_recovery: bool = False,
812 *,
541 - backend: Literal["opencv", "pyav"] = "opencv",
813 + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
814 **kwargs,
815 ) -> tuple[npt.NDArray, dict[str, Any]]:
816 """Load sampled frames from raw video bytes.
@@ -551,7 +823,8 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
823 dynamic subclass; ignored here.
824 frame_recovery: Enable forward-scan recovery for failed frames.
825 Only honored by the OpenCV codec.
554 - backend: Decoding codec — ``"opencv"`` or ``"pyav"`` .
826 + backend: Decoding codec — ``"opencv"``, ``"pyav"``, or
827 + ``"pynvvideocodec"``.
828
829 Returns:
830 Tuple of ``(frames_array, metadata_dict)``.
@@ -584,10 +857,21 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
857 frames, valid = cls.decode_frames(
858 container, frame_idx, source.original_fps, source.duration
859 )
860 + elif backend == PYNVVIDEOCODEC_VIDEO_BACKEND:
861 + if frame_recovery:
862 + raise ValueError(
863 + "frame_recovery is not supported for "
864 + f"`{PYNVVIDEOCODEC_VIDEO_BACKEND}` backend"
865 + )
866 + frames, source, frame_idx, valid = cls.decode_frames_pynvvideocodec(
867 + data,
868 + target,
869 + **kwargs,
870 + )
871 else:
872 raise ValueError(
873 f"Unknown video codec backend {backend!r}; "
590 - "valid options: 'opencv', 'pyav'."
874 + "valid options: 'opencv', 'pyav', 'pynvvideocodec'."
875 )
876
877 if len(valid) < len(frame_idx):
@@ -605,6 +889,40 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
889 )
890
891
892 +@VIDEO_LOADER_REGISTRY.register(PYNVVIDEOCODEC_VIDEO_BACKEND)
893 +class PyNvVideoCodecVideoBackend(VideoBackend):
894 + """Hardware-accelerated video backend using PyNvVideoCodec.
895 +
896 + The backend first opens the stream only to read metadata and compute the
897 + sampled frame indices. It then acquires the raw decoded RGB byte count from
898 + the process-local multimodal GPU memory pool before decoding the selected
899 + frames into VRAM. Decoded frames are copied into pinned host memory before
900 + the lease is released, so downstream preprocessing continues to receive a
901 + CPU ``np.ndarray`` in NHWC RGB format.
902 + """
903 +
904 + @classmethod
905 + def load_bytes(
906 + cls,
907 + data: bytes,
908 + num_frames: int = -1,
909 + fps: int = -1,
910 + max_duration: int = 300,
911 + frame_recovery: bool = False,
912 + **kwargs,
913 + ) -> tuple[npt.NDArray, dict[str, Any]]:
914 + kwargs.pop("backend", None)
915 + return super().load_bytes(
916 + data,
917 + num_frames=num_frames,
918 + fps=fps,
919 + max_duration=max_duration,
920 + frame_recovery=frame_recovery,
921 + backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
922 + **kwargs,
923 + )
924 +
925 +
926 @VIDEO_LOADER_REGISTRY.register(
927 "qwen3_vl",
928 video_processor="Qwen3VLVideoProcessor",
@@ -640,7 +958,7 @@ class Qwen3VLVideoBackend(VideoBackend):
958 max_duration: int = 300,
959 frame_recovery: bool = False,
960 *,
643 - backend: Literal["opencv", "pyav"] = "opencv",
961 + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
962 **kwargs,
963 ) -> tuple[npt.NDArray, dict[str, Any]]:
964 return super().load_bytes(
@@ -719,7 +1037,7 @@ class Qwen2VLVideoBackend(VideoBackend):
1037 max_duration: int = 300,
1038 frame_recovery: bool = False,
1039 *,
722 - backend: Literal["opencv", "pyav"] = "opencv",
1040 + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
1041 **kwargs,
1042 ) -> tuple[npt.NDArray, dict[str, Any]]:
1043 return super().load_bytes(
@@ -811,7 +1129,7 @@ class DynamicVideoBackend(VideoBackend):
1129 max_duration: int = 300,
1130 frame_recovery: bool = False,
1131 *,
814 - backend: Literal["opencv", "pyav"] = "opencv",
1132 + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
1133 **kwargs,
1134 ) -> tuple[npt.NDArray, dict[str, Any]]:
1135 return super().load_bytes(
@@ -936,7 +1254,7 @@ class GLM46VVideoBackend(VideoBackend):
1254 max_duration: int = 300,
1255 frame_recovery: bool = False,
1256 *,
939 - backend: Literal["opencv", "pyav"] = "opencv",
1257 + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
1258 **kwargs,
1259 ) -> tuple[npt.NDArray, dict[str, Any]]:
1260 return super().load_bytes(
@@ -1034,7 +1352,7 @@ class GLMGAVideoBackend(VideoBackend):
1352 max_duration: int = 300,
1353 frame_recovery: bool = False,
1354 *,
1037 - backend: Literal["opencv", "pyav"] = "opencv",
1355 + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
1356 **kwargs,
1357 ) -> tuple[npt.NDArray, dict[str, Any]]:
1358 frames, metadata = super().load_bytes(
@@ -1353,7 +1671,7 @@ class NemotronVLVideoBackend(VideoBackend):
1671 max_duration: int = 300,
1672 frame_recovery: bool = False,
1673 *,
1356 - backend: Literal["opencv", "pyav"] = "opencv",
1674 + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
1675 **kwargs,
1676 ) -> tuple[npt.NDArray, dict[str, Any]]:
1677 frames, metadata = super().load_bytes(
vllm/renderers/base.py
+11
@@ -29,6 +29,7 @@ from vllm.inputs import (
29 from vllm.logger import init_logger
30 from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry
31 from vllm.multimodal.cache import BaseMultiModalProcessorCache
32 +from vllm.multimodal.gpu_ipc_memory import maybe_init_mm_gpu_ipc_pool
33 from vllm.multimodal.parse import (
34 MultiModalDataItems,
35 MultiModalUUIDItems,
@@ -111,6 +112,16 @@ class BaseRenderer(ABC, Generic[_T]):
112 safe_load_prompt_embeds, executor=self._executor
113 )
114 if mm_registry.supports_multimodal_inputs(config.model_config):
115 + # Install the process-global GPU memory pool used to gate
116 + # frontend GPU-side multimodal decoding (no-op when the budget
117 + # is 0). Lives in the API-server process only.
118 + mm_config = config.model_config.multimodal_config
119 + if mm_config is not None:
120 + maybe_init_mm_gpu_ipc_pool(
121 + mm_config.mm_ipc_gpu_memory_gb,
122 + config.parallel_config._api_process_count,
123 + )
124 +
125 mm_processor_cache = mm_registry.processor_cache_from_config(config)
126
127 with set_default_torch_num_threads():
vllm/v1/worker/gpu_worker.py
+82 -2
@@ -50,6 +50,12 @@ from vllm.distributed.weight_transfer import (
50 from vllm.logger import init_logger
51 from vllm.lora.request import LoRARequest
52 from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
53 +from vllm.multimodal.video import (
54 + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
55 + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
56 + PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
57 + PYNVVIDEOCODEC_VIDEO_BACKEND,
58 +)
59 from vllm.platforms import current_platform
60 from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper
61 from vllm.sequence import IntermediateTensors
@@ -428,7 +434,7 @@ class Worker(WorkerBase):
434 "correspondingly."
435 )
436 logger.info(msg)
431 - return kv_cache_memory_bytes
437 + return self._reserve_mm_ipc_gpu_memory(kv_cache_memory_bytes)
438
439 # Execute a forward pass with dummy inputs to profile the memory usage
440 # of the model.
@@ -550,7 +556,81 @@ class Worker(WorkerBase):
556 suggested_util,
557 )
558
553 - return int(self.available_kv_cache_memory_bytes)
559 + return self._reserve_mm_ipc_gpu_memory(
560 + int(self.available_kv_cache_memory_bytes)
561 + )
562 +
563 + @staticmethod
564 + def _uses_pynvvideocodec_video_backend(mm_config) -> bool:
565 + video_kwargs = mm_config.media_io_kwargs.get("video", {})
566 + video_loader_backend = (
567 + video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
568 + )
569 + codec_backend = video_kwargs.get("backend")
570 + return (
571 + video_loader_backend == PYNVVIDEOCODEC_VIDEO_BACKEND
572 + or codec_backend == PYNVVIDEOCODEC_VIDEO_BACKEND
573 + )
574 +
575 + def _reserve_mm_ipc_gpu_memory(self, available_kv_cache_memory_bytes: int) -> int:
576 + """Carve frontend multimodal GPU memory out of the KV cache.
577 +
578 + The frontend (API-server) process allocates GPU memory for hardware
579 + multimodal decoding. Raw decoded frames are bounded by
580 + ``mm_ipc_gpu_memory_gb`` and acquired by the frontend semaphore. Some
581 + decoders also keep persistent surfaces around; reserve a fixed upper
582 + bound for those when the corresponding backend is configured.
583 + """
584 + mm_config = self.model_config.multimodal_config
585 + if mm_config is None:
586 + return available_kv_cache_memory_bytes
587 +
588 + raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes)
589 + # Each api_server_count process runs its OWN decoder surfaces + NVDEC/CUVID
590 + # CUDA context on the GPU, outside this (worker) memory pool. Reserve that
591 + # per-server footprint x api_server_count so gpu_memory_utilization bounds
592 + # TOTAL GPU usage across all API-server processes. Without the multiply,
593 + # HW decode overshoots the budget by ~(api_server_count-1) x per-server and
594 + # OOMs at high gmu, while SW decode (no per-server GPU allocation) does not.
595 + num_api_servers = max(1, getattr(self.parallel_config, "_api_process_count", 1))
596 + per_server_decoder_bytes = (
597 + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES
598 + * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
599 + + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
600 + )
601 + decoder_reserved_bytes = (
602 + num_api_servers * per_server_decoder_bytes
603 + if self._uses_pynvvideocodec_video_backend(mm_config)
604 + else 0
605 + )
606 + reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes
607 + if reserved_bytes <= 0:
608 + return available_kv_cache_memory_bytes
609 +
610 + remaining = available_kv_cache_memory_bytes - reserved_bytes
611 + if remaining <= 0:
612 + raise ValueError(
613 + f"frontend multimodal GPU decoding reserves "
614 + f"{format_gib(reserved_bytes)} GiB "
615 + f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, "
616 + f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), "
617 + f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is "
618 + "available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a "
619 + "different video backend, or increase gpu_memory_utilization."
620 + )
621 + logger.info_once(
622 + "Reserving %s GiB of GPU memory for frontend multimodal decoding "
623 + "(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context "
624 + "across %d API server(s) @ %s GiB/server); "
625 + "KV cache memory reduced to %s GiB.",
626 + format_gib(reserved_bytes),
627 + format_gib(raw_frame_reserved_bytes),
628 + format_gib(decoder_reserved_bytes),
629 + num_api_servers,
630 + format_gib(per_server_decoder_bytes),
631 + format_gib(remaining),
632 + )
633 + return remaining
634
635 def get_kv_connector_handshake_metadata(
636 self,