| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Admission control for frontend GPU-side multimodal work. |
| 4 | |
| 5 | When multimodal media is decoded on the GPU in the API-server (frontend) |
| 6 | process, the decoded buffers compete for the same device memory that the |
| 7 | engine reserves for weights, activations, and the KV cache. To keep the |
| 8 | frontend's GPU usage within a sequestered budget (see |
| 9 | ``MultiModalConfig.mm_ipc_gpu_memory_gb``), decode paths acquire the number of |
| 10 | bytes they need from a process-global :class:`MultiModalGPUMemoryPool` before |
| 11 | allocating on the device and release them once the device memory is freed. |
| 12 | |
| 13 | The pool is a simple byte-counting semaphore: ``acquire`` blocks until enough |
| 14 | budget is free, so concurrent requests serialize rather than oversubscribe the |
| 15 | GPU. It lives only in the frontend process; the engine carves the matching |
| 16 | amount out of its KV-cache budget so the headroom physically exists. |
| 17 | """ |
| 18 | |
| 19 | import threading |
| 20 | from typing import TYPE_CHECKING |
| 21 | |
| 22 | from vllm.logger import init_logger |
| 23 | from vllm.utils.mem_constants import GiB_bytes |
| 24 | from vllm.utils.mem_utils import format_gib |
| 25 | |
| 26 | if TYPE_CHECKING: |
| 27 | from vllm.config.multimodal import MultiModalConfig |
| 28 | |
| 29 | logger = init_logger(__name__) |
| 30 | |
| 31 | |
| 32 | class MultiModalGPUMemoryLease: |
| 33 | """A handle for bytes acquired from a :class:`MultiModalGPUMemoryPool`. |
| 34 | |
| 35 | Releasing is idempotent and the lease doubles as a context manager so the |
| 36 | budget is returned even if the decode raises. |
| 37 | """ |
| 38 | |
| 39 | def __init__(self, pool: "MultiModalGPUMemoryPool", lease_id: int, nbytes: int): |
| 40 | self.lease_id = lease_id |
| 41 | self.nbytes = nbytes |
| 42 | self._pool = pool |
| 43 | |
| 44 | def release(self) -> None: |
| 45 | self._pool._release(self) |
| 46 | |
| 47 | def __enter__(self) -> "MultiModalGPUMemoryLease": |
| 48 | return self |
| 49 | |
| 50 | def __exit__(self, *exc_info) -> None: |
| 51 | self.release() |
| 52 | |
| 53 | |
| 54 | class MultiModalGPUMemoryPool: |
| 55 | """Blocking byte-counting semaphore for frontend GPU multimodal memory. |
| 56 | |
| 57 | Thread-safe in both directions: ``acquire`` (blocking) and ``release`` are |
| 58 | typically called from the renderer's multimodal executor threads. |
| 59 | """ |
| 60 | |
| 61 | def __init__(self, total_bytes: int): |
| 62 | if total_bytes <= 0: |
| 63 | raise ValueError(f"total_bytes must be positive, got {total_bytes}") |
| 64 | self._total_bytes = total_bytes |
| 65 | self._available = total_bytes |
| 66 | self._cond = threading.Condition() |
| 67 | self._next_lease_id = 0 |
| 68 | # Outstanding lease ids, so a double release is a no-op. |
| 69 | self._outstanding: set[int] = set() |
| 70 | |
| 71 | @property |
| 72 | def total_bytes(self) -> int: |
| 73 | return self._total_bytes |
| 74 | |
| 75 | @property |
| 76 | def available_bytes(self) -> int: |
| 77 | with self._cond: |
| 78 | return self._available |
| 79 | |
| 80 | def acquire(self, nbytes: int) -> MultiModalGPUMemoryLease: |
| 81 | """Reserve ``nbytes``, blocking until that much budget is free. |
| 82 | |
| 83 | Raises ``ValueError`` if ``nbytes`` exceeds the pool's total capacity, |
| 84 | since such a request could never be satisfied. |
| 85 | """ |
| 86 | if nbytes < 0: |
| 87 | raise ValueError(f"Cannot acquire negative bytes: {nbytes}") |
| 88 | if nbytes > self._total_bytes: |
| 89 | raise ValueError( |
| 90 | f"Multimodal GPU decode requested {nbytes} bytes, which exceeds " |
| 91 | f"the total pool size of {self._total_bytes} bytes. Increase " |
| 92 | f"--mm-ipc-gpu-memory-gb or reduce the multimodal input size." |
| 93 | ) |
| 94 | with self._cond: |
| 95 | while self._available < nbytes: |
| 96 | self._cond.wait() |
| 97 | self._available -= nbytes |
| 98 | lease_id = self._next_lease_id |
| 99 | self._next_lease_id += 1 |
| 100 | self._outstanding.add(lease_id) |
| 101 | return MultiModalGPUMemoryLease(self, lease_id, nbytes) |
| 102 | |
| 103 | def _release(self, lease: MultiModalGPUMemoryLease) -> None: |
| 104 | with self._cond: |
| 105 | if lease.lease_id not in self._outstanding: |
| 106 | # Already released — idempotent. |
| 107 | return |
| 108 | self._outstanding.discard(lease.lease_id) |
| 109 | self._available += lease.nbytes |
| 110 | self._cond.notify_all() |
| 111 | |
| 112 | |
| 113 | _GLOBAL_POOL: MultiModalGPUMemoryPool | None = None |
| 114 | |
| 115 | |
| 116 | def set_mm_gpu_ipc_pool(pool: MultiModalGPUMemoryPool | None) -> None: |
| 117 | """Install the process-global pool (frontend process only).""" |
| 118 | global _GLOBAL_POOL |
| 119 | _GLOBAL_POOL = pool |
| 120 | |
| 121 | |
| 122 | def get_mm_gpu_ipc_pool() -> MultiModalGPUMemoryPool | None: |
| 123 | """Return the process-global pool, or ``None`` when gating is disabled.""" |
| 124 | return _GLOBAL_POOL |
| 125 | |
| 126 | |
| 127 | def maybe_init_mm_gpu_ipc_pool( |
| 128 | mm_ipc_gpu_memory_gb: float, |
| 129 | api_process_count: int = 1, |
| 130 | ) -> MultiModalGPUMemoryPool | None: |
| 131 | """Create and install the global pool from the configured GiB budget. |
| 132 | |
| 133 | Returns ``None`` (and leaves gating disabled) when the budget is 0. When |
| 134 | multiple API-server processes share one engine, each process gets an equal |
| 135 | slice of the user-provided frontend budget. |
| 136 | """ |
| 137 | if mm_ipc_gpu_memory_gb <= 0: |
| 138 | set_mm_gpu_ipc_pool(None) |
| 139 | return None |
| 140 | if api_process_count <= 0: |
| 141 | raise ValueError(f"api_process_count must be positive, got {api_process_count}") |
| 142 | total_bytes = int(mm_ipc_gpu_memory_gb * GiB_bytes) // api_process_count |
| 143 | pool = MultiModalGPUMemoryPool(total_bytes) |
| 144 | set_mm_gpu_ipc_pool(pool) |
| 145 | logger.info( |
| 146 | "Initialized multimodal GPU IPC memory pool with %d bytes for this API " |
| 147 | "process (%.2f GiB total budget across %d API process(es)).", |
| 148 | total_bytes, |
| 149 | mm_ipc_gpu_memory_gb, |
| 150 | api_process_count, |
| 151 | ) |
| 152 | return pool |
| 153 | |
| 154 | |
| 155 | def reserve_mm_ipc_gpu_memory( |
| 156 | available_kv_cache_memory_bytes: int, |
| 157 | mm_config: "MultiModalConfig | None", |
| 158 | api_process_count: int = 1, |
| 159 | ) -> int: |
| 160 | """Return KV-cache memory remaining after frontend multimodal reservations. |
| 161 | |
| 162 | The reservation covers: |
| 163 | |
| 164 | * The total ``mm_ipc_gpu_memory_gb`` budget for transient decoded-frame |
| 165 | buffers. This budget is divided among API processes, so it is not |
| 166 | multiplied by ``api_process_count``. |
| 167 | * For GPU video backends, a fixed upper bound for each API process's |
| 168 | retained decoder surfaces and CUDA context. The PyNvVideoCodec surface |
| 169 | reservation scales with its configured ``hw_decoders`` value, and the |
| 170 | entire decoder reservation scales with ``api_process_count`` because |
| 171 | these resources are not shared between processes. |
| 172 | |
| 173 | Args: |
| 174 | available_kv_cache_memory_bytes: KV-cache capacity before reserving |
| 175 | memory for frontend multimodal processing. |
| 176 | mm_config: Multimodal configuration, or ``None`` when multimodal |
| 177 | processing is disabled. |
| 178 | api_process_count: Number of frontend API processes sharing the GPU. |
| 179 | Values below one are treated as one. |
| 180 | |
| 181 | Returns: |
| 182 | KV-cache capacity after subtracting the frontend reservation. |
| 183 | |
| 184 | Raises: |
| 185 | ValueError: If the reservation leaves no memory for the KV cache. |
| 186 | |
| 187 | """ |
| 188 | if mm_config is None: |
| 189 | return available_kv_cache_memory_bytes |
| 190 | |
| 191 | from vllm import envs |
| 192 | from vllm.multimodal.video_decoders import ( |
| 193 | PYNVVIDEOCODEC_DEFAULT_HW_DECODERS, |
| 194 | PYNVVIDEOCODEC_VIDEO_BACKEND, |
| 195 | ) |
| 196 | from vllm.multimodal.video_decoders.pynvvideocodec import ( |
| 197 | PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES, |
| 198 | PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES, |
| 199 | validate_pynvvideocodec_hw_decoders, |
| 200 | ) |
| 201 | |
| 202 | raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes) |
| 203 | # Each API server process runs its own decoder surfaces and NVDEC/CUVID CUDA |
| 204 | # context on the GPU, outside the worker memory pool. Reserve that footprint |
| 205 | # per process so gpu_memory_utilization bounds total GPU usage across them. |
| 206 | num_api_servers = max(1, api_process_count) |
| 207 | video_kwargs = mm_config.media_io_kwargs.get("video", {}) |
| 208 | video_loader_backend = ( |
| 209 | video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND |
| 210 | ) |
| 211 | codec_backend = video_kwargs.get("backend") |
| 212 | uses_pynvvideocodec = ( |
| 213 | video_loader_backend == PYNVVIDEOCODEC_VIDEO_BACKEND |
| 214 | or codec_backend == PYNVVIDEOCODEC_VIDEO_BACKEND |
| 215 | ) |
| 216 | hw_decoders = ( |
| 217 | validate_pynvvideocodec_hw_decoders( |
| 218 | video_kwargs.get("hw_decoders", PYNVVIDEOCODEC_DEFAULT_HW_DECODERS) |
| 219 | ) |
| 220 | if uses_pynvvideocodec |
| 221 | else 1 |
| 222 | ) |
| 223 | per_server_decoder_bytes = ( |
| 224 | PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * hw_decoders |
| 225 | + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES |
| 226 | ) |
| 227 | decoder_reserved_bytes = ( |
| 228 | num_api_servers * per_server_decoder_bytes |
| 229 | if mm_config.use_gpu_video_backend() |
| 230 | else 0 |
| 231 | ) |
| 232 | reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes |
| 233 | if reserved_bytes <= 0: |
| 234 | return available_kv_cache_memory_bytes |
| 235 | |
| 236 | remaining = available_kv_cache_memory_bytes - reserved_bytes |
| 237 | if remaining <= 0: |
| 238 | raise ValueError( |
| 239 | f"frontend multimodal GPU decoding reserves " |
| 240 | f"{format_gib(reserved_bytes)} GiB " |
| 241 | f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, " |
| 242 | f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), " |
| 243 | f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is " |
| 244 | "available for the KV cache. Reduce mm_ipc_gpu_memory_gb or " |
| 245 | "hw_decoders, use a different video backend, or increase " |
| 246 | "gpu_memory_utilization." |
| 247 | ) |
| 248 | logger.info_once( |
| 249 | "Reserving %s GiB of GPU memory for frontend multimodal decoding " |
| 250 | "(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context " |
| 251 | "across %d API server(s) @ %s GiB/server); " |
| 252 | "KV cache memory reduced to %s GiB.", |
| 253 | format_gib(reserved_bytes), |
| 254 | format_gib(raw_frame_reserved_bytes), |
| 255 | format_gib(decoder_reserved_bytes), |
| 256 | num_api_servers, |
| 257 | format_gib(per_server_decoder_bytes), |
| 258 | format_gib(remaining), |
| 259 | ) |
| 260 | return remaining |