[Misc] Move PyNvVideoCodec stuff out of gpu worker (#49322)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Isotr0py committed
Jul 22, 2026 at 10:11 UTC
1750e443f26c5038b91a73f53f5a914adc000b88
6 files changed
+200
-190
tests/config/test_multimodal_config.py
+9
@@ -43,6 +43,15 @@ def test_language_model_only_affects_model_hash():
43
assert base_hash != lm_only_hash
44
45
46
+@pytest.mark.parametrize("backend_arg", ["video_backend", "backend"])
47
+def test_use_gpu_video_backend_from_media_io_kwargs(backend_arg: str):
48
+ config = MultiModalConfig(
49
+ media_io_kwargs={"video": {backend_arg: "pynvvideocodec"}}
50
+ )
51
+
52
+ assert config.use_gpu_video_backend()
53
+
54
+
55
def test_mm_encoder_fp8_scale_path_requires_fp8():
56
with pytest.raises(ValueError, match="mm_encoder_attn_dtype"):
57
MultiModalConfig(mm_encoder_fp8_scale_path="/tmp/scales.json")
tests/multimodal/test_gpu_ipc_memory.py
+97
@@ -6,15 +6,43 @@ import time
6
7
import pytest
8
9
+import vllm.config.multimodal as multimodal_config_module
10
+from vllm.config.multimodal import MultiModalConfig
11
from vllm.multimodal.gpu_ipc_memory import (
12
MultiModalGPUMemoryPool,
13
get_mm_gpu_ipc_pool,
14
maybe_init_mm_gpu_ipc_pool,
15
+ reserve_mm_ipc_gpu_memory,
16
set_mm_gpu_ipc_pool,
17
)
18
+from vllm.multimodal.video import (
19
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
20
+ PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
21
+ PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
22
+ PYNVVIDEOCODEC_VIDEO_BACKEND,
23
+)
24
from vllm.utils.mem_constants import GiB_bytes
25
26
27
+def _mm_config(
28
+ *,
29
+ mm_ipc_gpu_memory_gb: float = 0,
30
+ video_backend: str | None = None,
31
+) -> MultiModalConfig:
32
+ video_kwargs = {} if video_backend is None else {"video_backend": video_backend}
33
+ return MultiModalConfig(
34
+ mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
35
+ media_io_kwargs={"video": video_kwargs} if video_kwargs else {},
36
+ )
37
+
38
+
39
+def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:
40
+ return api_process_count * (
41
+ PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
42
+ + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
43
+ )
44
+
45
+
46
def test_acquire_release_accounting():
47
pool = MultiModalGPUMemoryPool(total_bytes=100)
48
assert pool.available_bytes == 100
@@ -143,3 +171,72 @@ def test_global_pool_splits_budget_across_api_processes():
171
def test_global_pool_rejects_invalid_api_process_count():
172
with pytest.raises(ValueError):
173
maybe_init_mm_gpu_ipc_pool(2, api_process_count=0)
174
+
175
+
176
+@pytest.mark.parametrize("video_backend", [None, "opencv"])
177
+def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
178
+ monkeypatch: pytest.MonkeyPatch,
179
+ video_backend: str | None,
180
+):
181
+ monkeypatch.setattr(
182
+ multimodal_config_module.envs,
183
+ "VLLM_VIDEO_LOADER_BACKEND",
184
+ "opencv",
185
+ )
186
+ mm_config = _mm_config(
187
+ mm_ipc_gpu_memory_gb=0.25,
188
+ video_backend=video_backend,
189
+ )
190
+
191
+ assert reserve_mm_ipc_gpu_memory(GiB_bytes, mm_config) == int(0.75 * GiB_bytes)
192
+
193
+
194
+def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
195
+ monkeypatch: pytest.MonkeyPatch,
196
+):
197
+ monkeypatch.setattr(
198
+ multimodal_config_module.envs,
199
+ "VLLM_VIDEO_LOADER_BACKEND",
200
+ "opencv",
201
+ )
202
+ mm_config = _mm_config(
203
+ mm_ipc_gpu_memory_gb=0.25,
204
+ video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
205
+ )
206
+ available_bytes = 4 * GiB_bytes
207
+
208
+ assert reserve_mm_ipc_gpu_memory(available_bytes, mm_config) == (
209
+ available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget()
210
+ )
211
+
212
+
213
+def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
214
+ monkeypatch: pytest.MonkeyPatch,
215
+):
216
+ monkeypatch.setattr(
217
+ multimodal_config_module.envs,
218
+ "VLLM_VIDEO_LOADER_BACKEND",
219
+ PYNVVIDEOCODEC_VIDEO_BACKEND,
220
+ )
221
+ available_bytes = 4 * GiB_bytes
222
+
223
+ assert reserve_mm_ipc_gpu_memory(available_bytes, _mm_config()) == (
224
+ available_bytes - _pynvvideocodec_decoder_budget()
225
+ )
226
+
227
+
228
+def test_reserve_mm_ipc_gpu_memory_scales_decoder_budget_by_api_servers(
229
+ monkeypatch: pytest.MonkeyPatch,
230
+):
231
+ monkeypatch.setattr(
232
+ multimodal_config_module.envs,
233
+ "VLLM_VIDEO_LOADER_BACKEND",
234
+ PYNVVIDEOCODEC_VIDEO_BACKEND,
235
+ )
236
+ available_bytes = 8 * GiB_bytes
237
+
238
+ assert reserve_mm_ipc_gpu_memory(
239
+ available_bytes,
240
+ _mm_config(),
241
+ api_process_count=3,
242
+ ) == available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)
tests/v1/worker/test_gpu_worker.py
-109
@@ -6,122 +6,13 @@ from unittest.mock import patch
6
7
import pytest
8
9
-import vllm.v1.worker.gpu_worker as gpu_worker_module
10
-from vllm.multimodal.video import (
11
- PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
12
- PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
13
- PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
14
- PYNVVIDEOCODEC_VIDEO_BACKEND,
15
-)
9
from vllm.utils.mem_constants import GiB_bytes
10
from vllm.v1.worker import startup_plan
18
-from vllm.v1.worker.gpu_worker import Worker
11
from vllm.v1.worker.startup_plan import (
12
maybe_apply_startup_plan,
13
maybe_save_startup_plan,
14
)
15
24
-
25
-def _worker_with_mm_config(
26
- mm_config: SimpleNamespace,
27
- *,
28
- api_process_count: int = 1,
29
-) -> Worker:
30
- worker = object.__new__(Worker)
31
- worker.model_config = SimpleNamespace(multimodal_config=mm_config)
32
- worker.parallel_config = SimpleNamespace(_api_process_count=api_process_count)
33
- return worker
34
-
35
-
36
-def _mm_config(
37
- *,
38
- mm_ipc_gpu_memory_gb: float = 0,
39
- video_backend: str | None = None,
40
-) -> SimpleNamespace:
41
- video_kwargs = {} if video_backend is None else {"video_backend": video_backend}
42
- return SimpleNamespace(
43
- mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
44
- media_io_kwargs={"video": video_kwargs} if video_kwargs else {},
45
- )
46
-
47
-
48
-def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:
49
- return api_process_count * (
50
- PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
51
- + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
52
- )
53
-
54
-
55
-@pytest.mark.parametrize("video_backend", [None, "opencv"])
56
-def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
57
- monkeypatch: pytest.MonkeyPatch,
58
- video_backend: str | None,
59
-):
60
- monkeypatch.setattr(
61
- gpu_worker_module.envs,
62
- "VLLM_VIDEO_LOADER_BACKEND",
63
- "opencv",
64
- )
65
- worker = _worker_with_mm_config(
66
- _mm_config(mm_ipc_gpu_memory_gb=0.25, video_backend=video_backend)
67
- )
68
-
69
- assert worker._reserve_mm_ipc_gpu_memory(GiB_bytes) == int(0.75 * GiB_bytes)
70
-
71
-
72
-def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
73
- monkeypatch: pytest.MonkeyPatch,
74
-):
75
- monkeypatch.setattr(
76
- gpu_worker_module.envs,
77
- "VLLM_VIDEO_LOADER_BACKEND",
78
- "opencv",
79
- )
80
- worker = _worker_with_mm_config(
81
- _mm_config(
82
- mm_ipc_gpu_memory_gb=0.25,
83
- video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
84
- )
85
- )
86
- available_bytes = 4 * GiB_bytes
87
-
88
- assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
89
- available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget()
90
- )
91
-
92
-
93
-def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
94
- monkeypatch: pytest.MonkeyPatch,
95
-):
96
- monkeypatch.setattr(
97
- gpu_worker_module.envs,
98
- "VLLM_VIDEO_LOADER_BACKEND",
99
- PYNVVIDEOCODEC_VIDEO_BACKEND,
100
- )
101
- worker = _worker_with_mm_config(_mm_config())
102
- available_bytes = 4 * GiB_bytes
103
-
104
- assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
105
- available_bytes - _pynvvideocodec_decoder_budget()
106
- )
107
-
108
-
109
-def test_reserve_mm_ipc_gpu_memory_scales_pynvvideocodec_budget_by_api_servers(
110
- monkeypatch: pytest.MonkeyPatch,
111
-):
112
- monkeypatch.setattr(
113
- gpu_worker_module.envs,
114
- "VLLM_VIDEO_LOADER_BACKEND",
115
- PYNVVIDEOCODEC_VIDEO_BACKEND,
116
- )
117
- worker = _worker_with_mm_config(_mm_config(), api_process_count=3)
118
- available_bytes = 8 * GiB_bytes
119
-
120
- assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
121
- available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)
122
- )
123
-
124
-
16
# Startup-plan persistence (vllm/v1/worker/startup_plan.py), applied and
17
# saved by Worker.determine_available_memory / compile_or_warm_up_model.
18
vllm/config/multimodal.py
+15
@@ -8,6 +8,7 @@ from typing import Any, Literal, TypeAlias, TypedDict, final
8
from pydantic import ConfigDict, Field, field_validator, model_validator
9
from pydantic.dataclasses import dataclass
10
11
+import vllm.envs as envs
12
from vllm.config.utils import config
13
from vllm.utils.hashing import safe_hash
14
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -344,5 +345,19 @@ class MultiModalConfig:
345
kwargs = self.mm_processor_kwargs or {}
346
return kwargs | dict(inference_kwargs)
347
348
+ def use_gpu_video_backend(self) -> bool:
349
+ """Return whether the configured video loader or codec uses the GPU."""
350
+ from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
351
+
352
+ video_kwargs = self.media_io_kwargs.get("video", {})
353
+ video_loader_backend = (
354
+ video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
355
+ )
356
+ codec_backend = video_kwargs.get("backend")
357
+ return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or (
358
+ codec_backend is not None
359
+ and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend)
360
+ )
361
+
362
def is_multimodal_pruning_enabled(self):
363
return self.video_pruning_rate is not None and self.video_pruning_rate > 0
vllm/multimodal/gpu_ipc_memory.py
+69
@@ -17,9 +17,14 @@ amount out of its KV-cache budget so the headroom physically exists.
17
"""
18
19
import threading
20
+from typing import TYPE_CHECKING
21
22
from vllm.logger import init_logger
23
from vllm.utils.mem_constants import GiB_bytes
24
+from vllm.utils.mem_utils import format_gib
25
+
26
+if TYPE_CHECKING:
27
+ from vllm.config.multimodal import MultiModalConfig
28
29
logger = init_logger(__name__)
30
@@ -145,3 +150,67 @@ def maybe_init_mm_gpu_ipc_pool(
150
api_process_count,
151
)
152
return pool
153
+
154
+
155
+def reserve_mm_ipc_gpu_memory(
156
+ available_kv_cache_memory_bytes: int,
157
+ mm_config: "MultiModalConfig | None",
158
+ api_process_count: int = 1,
159
+) -> int:
160
+ """Carve frontend multimodal GPU memory out of the KV cache.
161
+
162
+ Raw decoded frames are bounded by ``mm_ipc_gpu_memory_gb`` and acquired by
163
+ the frontend semaphore. Some decoders also keep persistent surfaces around;
164
+ reserve a fixed upper bound for those when a GPU backend is configured.
165
+ """
166
+ if mm_config is None:
167
+ return available_kv_cache_memory_bytes
168
+
169
+ from vllm.multimodal.video import (
170
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
171
+ PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
172
+ PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
173
+ )
174
+
175
+ raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes)
176
+ # Each API server process runs its own decoder surfaces and NVDEC/CUVID CUDA
177
+ # context on the GPU, outside the worker memory pool. Reserve that footprint
178
+ # per process so gpu_memory_utilization bounds total GPU usage across them.
179
+ num_api_servers = max(1, api_process_count)
180
+ per_server_decoder_bytes = (
181
+ PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
182
+ + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
183
+ )
184
+ decoder_reserved_bytes = (
185
+ num_api_servers * per_server_decoder_bytes
186
+ if mm_config.use_gpu_video_backend()
187
+ else 0
188
+ )
189
+ reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes
190
+ if reserved_bytes <= 0:
191
+ return available_kv_cache_memory_bytes
192
+
193
+ remaining = available_kv_cache_memory_bytes - reserved_bytes
194
+ if remaining <= 0:
195
+ raise ValueError(
196
+ f"frontend multimodal GPU decoding reserves "
197
+ f"{format_gib(reserved_bytes)} GiB "
198
+ f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, "
199
+ f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), "
200
+ f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is "
201
+ "available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a "
202
+ "different video backend, or increase gpu_memory_utilization."
203
+ )
204
+ logger.info_once(
205
+ "Reserving %s GiB of GPU memory for frontend multimodal decoding "
206
+ "(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context "
207
+ "across %d API server(s) @ %s GiB/server); "
208
+ "KV cache memory reduced to %s GiB.",
209
+ format_gib(reserved_bytes),
210
+ format_gib(raw_frame_reserved_bytes),
211
+ format_gib(decoder_reserved_bytes),
212
+ num_api_servers,
213
+ format_gib(per_server_decoder_bytes),
214
+ format_gib(remaining),
215
+ )
216
+ return remaining
vllm/v1/worker/gpu_worker.py
+10
-81
@@ -51,12 +51,7 @@ from vllm.distributed.weight_transfer import (
51
from vllm.logger import init_logger
52
from vllm.lora.request import LoRARequest
53
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
54
-from vllm.multimodal.video import (
55
- PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
56
- PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
57
- PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
58
- VIDEO_LOADER_REGISTRY,
59
-)
54
+from vllm.multimodal.gpu_ipc_memory import reserve_mm_ipc_gpu_memory
55
from vllm.platforms import current_platform
56
from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper
57
from vllm.sequence import IntermediateTensors
@@ -478,7 +473,11 @@ class Worker(WorkerBase):
473
"correspondingly."
474
)
475
logger.info(msg)
481
- return self._reserve_mm_ipc_gpu_memory(kv_cache_memory_bytes)
476
+ return reserve_mm_ipc_gpu_memory(
477
+ kv_cache_memory_bytes,
478
+ self.model_config.multimodal_config,
479
+ getattr(self.parallel_config, "_api_process_count", 1),
480
+ )
481
482
# Execute a forward pass with dummy inputs to profile the memory usage
483
# of the model.
@@ -589,81 +588,11 @@ class Worker(WorkerBase):
588
suggested_util,
589
)
590
592
- return self._reserve_mm_ipc_gpu_memory(
593
- int(self.available_kv_cache_memory_bytes)
594
- )
595
-
596
- @staticmethod
597
- def _uses_gpu_video_backend(mm_config) -> bool:
598
- video_kwargs = mm_config.media_io_kwargs.get("video", {})
599
- video_loader_backend = (
600
- video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
601
- )
602
- codec_backend = video_kwargs.get("backend")
603
- return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or (
604
- codec_backend is not None
605
- and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend)
606
- )
607
-
608
- def _reserve_mm_ipc_gpu_memory(self, available_kv_cache_memory_bytes: int) -> int:
609
- """Carve frontend multimodal GPU memory out of the KV cache.
610
-
611
- The frontend (API-server) process allocates GPU memory for hardware
612
- multimodal decoding. Raw decoded frames are bounded by
613
- ``mm_ipc_gpu_memory_gb`` and acquired by the frontend semaphore. Some
614
- decoders also keep persistent surfaces around; reserve a fixed upper
615
- bound for those when the corresponding backend is configured.
616
- """
617
- mm_config = self.model_config.multimodal_config
618
- if mm_config is None:
619
- return available_kv_cache_memory_bytes
620
-
621
- raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes)
622
- # Each api_server_count process runs its OWN decoder surfaces + NVDEC/CUVID
623
- # CUDA context on the GPU, outside this (worker) memory pool. Reserve that
624
- # per-server footprint x api_server_count so gpu_memory_utilization bounds
625
- # TOTAL GPU usage across all API-server processes. Without the multiply,
626
- # HW decode overshoots the budget by ~(api_server_count-1) x per-server and
627
- # OOMs at high gmu, while SW decode (no per-server GPU allocation) does not.
628
- num_api_servers = max(1, getattr(self.parallel_config, "_api_process_count", 1))
629
- per_server_decoder_bytes = (
630
- PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES
631
- * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
632
- + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
633
- )
634
- decoder_reserved_bytes = (
635
- num_api_servers * per_server_decoder_bytes
636
- if self._uses_gpu_video_backend(mm_config)
637
- else 0
638
- )
639
- reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes
640
- if reserved_bytes <= 0:
641
- return available_kv_cache_memory_bytes
642
-
643
- remaining = available_kv_cache_memory_bytes - reserved_bytes
644
- if remaining <= 0:
645
- raise ValueError(
646
- f"frontend multimodal GPU decoding reserves "
647
- f"{format_gib(reserved_bytes)} GiB "
648
- f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, "
649
- f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), "
650
- f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is "
651
- "available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a "
652
- "different video backend, or increase gpu_memory_utilization."
653
- )
654
- logger.info_once(
655
- "Reserving %s GiB of GPU memory for frontend multimodal decoding "
656
- "(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context "
657
- "across %d API server(s) @ %s GiB/server); "
658
- "KV cache memory reduced to %s GiB.",
659
- format_gib(reserved_bytes),
660
- format_gib(raw_frame_reserved_bytes),
661
- format_gib(decoder_reserved_bytes),
662
- num_api_servers,
663
- format_gib(per_server_decoder_bytes),
664
- format_gib(remaining),
591
+ return reserve_mm_ipc_gpu_memory(
592
+ int(self.available_kv_cache_memory_bytes),
593
+ self.model_config.multimodal_config,
594
+ getattr(self.parallel_config, "_api_process_count", 1),
595
)
666
- return remaining
596
597
def get_kv_connector_handshake_metadata(
598
self,