| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Registered-memory allocation and residency for Mooncake transfers. |
| 4 | |
| 5 | The Producer pool stages source tensors in a registered slab. The Consumer |
| 6 | pool owns destination allocations from reservation through publication, |
| 7 | resident reuse, CUDA-safe retirement, and pressure-driven reclamation. |
| 8 | """ |
| 9 | |
| 10 | from __future__ import annotations |
| 11 | |
| 12 | import bisect |
| 13 | import threading |
| 14 | from collections import OrderedDict |
| 15 | from collections.abc import Callable |
| 16 | from dataclasses import dataclass |
| 17 | from typing import Generic, TypeVar |
| 18 | |
| 19 | import torch |
| 20 | |
| 21 | from vllm.distributed.ec_transfer.ec_connector.mooncake.transfer import ( |
| 22 | MooncakeTransfer, |
| 23 | ) |
| 24 | from vllm.logger import init_logger |
| 25 | |
| 26 | logger = init_logger(__name__) |
| 27 | |
| 28 | _T = TypeVar("_T") |
| 29 | |
| 30 | |
| 31 | @dataclass |
| 32 | class MemoryAllocation: |
| 33 | """Describe a tensor view carved from the Consumer receive slab.""" |
| 34 | |
| 35 | offset: int |
| 36 | size: int |
| 37 | tensor: torch.Tensor |
| 38 | |
| 39 | |
| 40 | @dataclass |
| 41 | class _ResidentEntry(Generic[_T]): |
| 42 | """Store one resident value and its ownership accounting.""" |
| 43 | |
| 44 | value: _T |
| 45 | pinned: bool = True |
| 46 | leases: int = 0 |
| 47 | |
| 48 | |
| 49 | @dataclass |
| 50 | class ResidentLease(Generic[_T]): |
| 51 | """Represent a borrow of one resident entry.""" |
| 52 | |
| 53 | key: str |
| 54 | _entry: _ResidentEntry[_T] |
| 55 | _active: bool = True |
| 56 | |
| 57 | @property |
| 58 | def value(self) -> _T: |
| 59 | return self._entry.value |
| 60 | |
| 61 | |
| 62 | class ContiguousAllocator: |
| 63 | """Own a registered slab and its aligned regions; callers serialize access.""" |
| 64 | |
| 65 | def __init__(self, capacity: int, alignment: int = 256): |
| 66 | self.alignment = alignment |
| 67 | self._capacity = capacity |
| 68 | self._free = [(0, capacity)] |
| 69 | self.tensor: torch.Tensor | None = None |
| 70 | self._disabled = False |
| 71 | |
| 72 | def prepare(self, device: torch.device, transfer: MooncakeTransfer) -> None: |
| 73 | if self.tensor is not None or self._disabled: |
| 74 | return |
| 75 | try: |
| 76 | tensor = torch.empty(self._capacity, dtype=torch.uint8, device=device) |
| 77 | ret = transfer.register_memory(tensor) |
| 78 | if ret != 0: |
| 79 | raise RuntimeError(f"Mooncake returned {ret}") |
| 80 | except (RuntimeError, torch.OutOfMemoryError) as error: |
| 81 | self._disabled = True |
| 82 | logger.warning("Could not initialize the EC registered buffer: %s", error) |
| 83 | return |
| 84 | self.tensor = tensor |
| 85 | self._free = [(0, tensor.nbytes)] |
| 86 | logger.info("Registered %d-byte buffer for Mooncake EC", tensor.nbytes) |
| 87 | |
| 88 | def view( |
| 89 | self, offset: int, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype |
| 90 | ) -> torch.Tensor: |
| 91 | assert self.tensor is not None |
| 92 | return self.tensor.narrow(0, offset, nbytes).view(dtype).view(shape) |
| 93 | |
| 94 | def close(self, transfer: MooncakeTransfer) -> bool: |
| 95 | if self.tensor is not None and not transfer.unregister_memory(self.tensor): |
| 96 | return False |
| 97 | self.tensor = None |
| 98 | self._free.clear() |
| 99 | return True |
| 100 | |
| 101 | def allocate(self, nbytes: int) -> tuple[int, int] | None: |
| 102 | size = (nbytes + self.alignment - 1) // self.alignment * self.alignment |
| 103 | for index, (offset, available) in enumerate(self._free): |
| 104 | if size > available: |
| 105 | continue |
| 106 | if size == available: |
| 107 | self._free.pop(index) |
| 108 | else: |
| 109 | self._free[index] = (offset + size, available - size) |
| 110 | return offset, size |
| 111 | return None |
| 112 | |
| 113 | def free(self, offset: int, size: int) -> None: |
| 114 | index = bisect.bisect_left(self._free, (offset, size)) |
| 115 | self._free.insert(index, (offset, size)) |
| 116 | if index + 1 < len(self._free): |
| 117 | next_offset, next_size = self._free[index + 1] |
| 118 | if offset + size == next_offset: |
| 119 | self._free[index] = (offset, size + next_size) |
| 120 | self._free.pop(index + 1) |
| 121 | if index > 0: |
| 122 | previous_offset, previous_size = self._free[index - 1] |
| 123 | current_offset, current_size = self._free[index] |
| 124 | if previous_offset + previous_size == current_offset: |
| 125 | self._free[index - 1] = ( |
| 126 | previous_offset, |
| 127 | previous_size + current_size, |
| 128 | ) |
| 129 | self._free.pop(index) |
| 130 | |
| 131 | |
| 132 | class ResidentPool(Generic[_T]): |
| 133 | """Track resident values, active leases, and LRU eviction eligibility.""" |
| 134 | |
| 135 | def __init__(self): |
| 136 | self._entries: dict[str, _ResidentEntry[_T]] = {} |
| 137 | self._evictable: OrderedDict[str, None] = OrderedDict() |
| 138 | |
| 139 | def referenced(self) -> list[str]: |
| 140 | return [key for key, entry in self._entries.items() if entry.pinned] |
| 141 | |
| 142 | def get(self, key: str) -> _T | None: |
| 143 | entry = self._entries.get(key) |
| 144 | return entry.value if entry is not None else None |
| 145 | |
| 146 | def insert( |
| 147 | self, |
| 148 | key: str, |
| 149 | value: _T, |
| 150 | ) -> _T | None: |
| 151 | """Pin an entry and return a displaced value that has no owners.""" |
| 152 | previous = self._entries.get(key) |
| 153 | entry = _ResidentEntry(value) |
| 154 | self._entries[key] = entry |
| 155 | self._evictable.pop(key, None) |
| 156 | if previous is not None and previous.leases == 0: |
| 157 | return previous.value |
| 158 | return None |
| 159 | |
| 160 | def pin(self, key: str) -> _T | None: |
| 161 | entry = self._entries.get(key) |
| 162 | if entry is None: |
| 163 | return None |
| 164 | self._evictable.pop(key, None) |
| 165 | entry.pinned = True |
| 166 | return entry.value |
| 167 | |
| 168 | def acquire(self, key: str) -> ResidentLease[_T] | None: |
| 169 | entry = self._entries.get(key) |
| 170 | if entry is None: |
| 171 | return None |
| 172 | entry.leases += 1 |
| 173 | self._evictable.pop(key, None) |
| 174 | return ResidentLease(key, entry) |
| 175 | |
| 176 | def release(self, lease: ResidentLease[_T]) -> _T | None: |
| 177 | if not lease._active: |
| 178 | return None |
| 179 | lease._active = False |
| 180 | entry = lease._entry |
| 181 | entry.leases -= 1 |
| 182 | current = self._entries.get(lease.key) |
| 183 | if current is not entry: |
| 184 | if entry.leases == 0: |
| 185 | return entry.value |
| 186 | return None |
| 187 | if not entry.pinned and entry.leases == 0: |
| 188 | self._evictable[lease.key] = None |
| 189 | return None |
| 190 | |
| 191 | def consume(self, lease: ResidentLease[_T]) -> tuple[_T, _T | None]: |
| 192 | current = self._entries[lease.key] |
| 193 | current.pinned = True |
| 194 | self._evictable.pop(lease.key, None) |
| 195 | released = self.release(lease) |
| 196 | return current.value, released |
| 197 | |
| 198 | def retire(self, key: str) -> None: |
| 199 | if key not in self._entries: |
| 200 | return |
| 201 | entry = self._entries[key] |
| 202 | entry.pinned = False |
| 203 | if entry.leases == 0: |
| 204 | self._evictable[key] = None |
| 205 | |
| 206 | def evict_lru(self, evict: Callable[[str, _T], bool]) -> str | None: |
| 207 | for key in list(self._evictable): |
| 208 | entry = self._entries[key] |
| 209 | if not evict(key, entry.value): |
| 210 | continue |
| 211 | self._evictable.pop(key, None) |
| 212 | del self._entries[key] |
| 213 | return key |
| 214 | return None |
| 215 | |
| 216 | def clear(self) -> None: |
| 217 | self._entries.clear() |
| 218 | self._evictable.clear() |
| 219 | |
| 220 | |
| 221 | @dataclass |
| 222 | class StagedSources: |
| 223 | """Own Producer tensor views and their staging-slab regions.""" |
| 224 | |
| 225 | tensors: list[torch.Tensor] |
| 226 | regions: list[tuple[int, int]] |
| 227 | |
| 228 | |
| 229 | class ProducerMemoryPool: |
| 230 | """Own the Producer staging slab and regions carved from it.""" |
| 231 | |
| 232 | def __init__(self, capacity: int, transfer: MooncakeTransfer) -> None: |
| 233 | self._transfer = transfer |
| 234 | self._allocator = ContiguousAllocator(capacity) |
| 235 | self._lock = threading.Lock() |
| 236 | self._local = threading.local() |
| 237 | |
| 238 | @property |
| 239 | def tensor(self) -> torch.Tensor | None: |
| 240 | return self._allocator.tensor |
| 241 | |
| 242 | def _free_regions(self, regions: list[tuple[int, int]]) -> None: |
| 243 | for offset, size in regions: |
| 244 | self._allocator.free(offset, size) |
| 245 | |
| 246 | def stage(self, tensors: list[torch.Tensor]) -> StagedSources | None: |
| 247 | """Copy tensors into one registered slab, or return None for fallback.""" |
| 248 | if not tensors: |
| 249 | return StagedSources([], []) |
| 250 | allocator = self._allocator |
| 251 | staged: list[torch.Tensor] = [] |
| 252 | regions: list[tuple[int, int]] = [] |
| 253 | with self._lock: |
| 254 | allocator.prepare(tensors[0].device, self._transfer) |
| 255 | pool = allocator.tensor |
| 256 | if pool is None: |
| 257 | return None |
| 258 | for tensor in tensors: |
| 259 | region = allocator.allocate(tensor.nbytes) |
| 260 | if region is None: |
| 261 | self._free_regions(regions) |
| 262 | return None |
| 263 | regions.append(region) |
| 264 | staged.append( |
| 265 | allocator.view( |
| 266 | region[0], tensor.nbytes, tuple(tensor.shape), tensor.dtype |
| 267 | ) |
| 268 | ) |
| 269 | if pool.device.type == "cuda": |
| 270 | stream = getattr(self._local, "stream", None) |
| 271 | if stream is None: |
| 272 | stream = torch.cuda.Stream(device=pool.device) |
| 273 | self._local.stream = stream |
| 274 | with torch.cuda.stream(stream): |
| 275 | for destination, source in zip(staged, tensors): |
| 276 | destination.copy_(source, non_blocking=True) |
| 277 | stream.synchronize() |
| 278 | else: |
| 279 | for destination, source in zip(staged, tensors): |
| 280 | destination.copy_(source) |
| 281 | return StagedSources(staged, regions) |
| 282 | |
| 283 | def release(self, staged: StagedSources) -> None: |
| 284 | if not staged.regions: |
| 285 | return |
| 286 | with self._lock: |
| 287 | self._free_regions(staged.regions) |
| 288 | |
| 289 | def close(self) -> None: |
| 290 | with self._lock: |
| 291 | self._allocator.close(self._transfer) |
| 292 | |
| 293 | |
| 294 | class ConsumerMemoryPool: |
| 295 | """Own the registered receive slab and resident allocation lifecycle.""" |
| 296 | |
| 297 | def __init__( |
| 298 | self, |
| 299 | capacity: int, |
| 300 | transfer: MooncakeTransfer, |
| 301 | ) -> None: |
| 302 | self._transfer = transfer |
| 303 | self._allocator = ContiguousAllocator(capacity) |
| 304 | self._residents: ResidentPool[MemoryAllocation] = ResidentPool() |
| 305 | self._retire_events: dict[str, torch.Event] = {} |
| 306 | self._pending_frees: list[tuple[torch.Event, MemoryAllocation]] = [] |
| 307 | self._reclaimed: set[str] = set() |
| 308 | self.lock = threading.RLock() |
| 309 | |
| 310 | @property |
| 311 | def tensor(self) -> torch.Tensor | None: |
| 312 | return self._allocator.tensor |
| 313 | |
| 314 | def prepare( |
| 315 | self, |
| 316 | device: torch.device, |
| 317 | ) -> None: |
| 318 | with self.lock: |
| 319 | self._allocator.prepare(device, self._transfer) |
| 320 | |
| 321 | def _free(self, allocation: MemoryAllocation) -> None: |
| 322 | self._allocator.free(allocation.offset, allocation.size) |
| 323 | |
| 324 | def free(self, allocation: MemoryAllocation) -> None: |
| 325 | with self.lock: |
| 326 | self._free(allocation) |
| 327 | |
| 328 | def _defer_or_free( |
| 329 | self, allocation: MemoryAllocation, event: torch.Event | None |
| 330 | ) -> None: |
| 331 | if event is None or event.query(): |
| 332 | self._free(allocation) |
| 333 | else: |
| 334 | self._pending_frees.append((event, allocation)) |
| 335 | |
| 336 | def _poll_frees_locked(self) -> None: |
| 337 | pending = [] |
| 338 | for event, allocation in self._pending_frees: |
| 339 | if event.query(): |
| 340 | self._free(allocation) |
| 341 | else: |
| 342 | pending.append((event, allocation)) |
| 343 | self._pending_frees = pending |
| 344 | |
| 345 | def _reclaim_locked(self, nbytes: int) -> tuple[int, int] | None: |
| 346 | def evict(mm_hash: str, allocation: MemoryAllocation) -> bool: |
| 347 | event = self._retire_events.pop(mm_hash, None) |
| 348 | self._defer_or_free(allocation, event) |
| 349 | self._reclaimed.add(mm_hash) |
| 350 | return True |
| 351 | |
| 352 | while self._residents.evict_lru(evict) is not None: |
| 353 | # Eviction defers any free whose CUDA event is still pending, so |
| 354 | # those bytes reach the allocator only once the event is polled. |
| 355 | self._poll_frees_locked() |
| 356 | region = self._allocator.allocate(nbytes) |
| 357 | if region is not None: |
| 358 | return region |
| 359 | self._poll_frees_locked() |
| 360 | return self._allocator.allocate(nbytes) |
| 361 | |
| 362 | def _make_allocation( |
| 363 | self, |
| 364 | region: tuple[int, int], |
| 365 | nbytes: int, |
| 366 | shape: tuple[int, ...], |
| 367 | dtype: torch.dtype, |
| 368 | ) -> MemoryAllocation: |
| 369 | offset, size = region |
| 370 | tensor = self._allocator.view(offset, nbytes, shape, dtype) |
| 371 | return MemoryAllocation(offset, size, tensor) |
| 372 | |
| 373 | def try_allocate( |
| 374 | self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype |
| 375 | ) -> MemoryAllocation | None: |
| 376 | with self.lock: |
| 377 | allocator = self._allocator |
| 378 | assert allocator.tensor is not None |
| 379 | self._poll_frees_locked() |
| 380 | region = allocator.allocate(nbytes) |
| 381 | if region is None: |
| 382 | return None |
| 383 | return self._make_allocation(region, nbytes, shape, dtype) |
| 384 | |
| 385 | def reclaim_and_allocate( |
| 386 | self, nbytes: int, shape: tuple[int, ...], dtype: torch.dtype |
| 387 | ) -> MemoryAllocation | None: |
| 388 | with self.lock: |
| 389 | region = self._reclaim_locked(nbytes) |
| 390 | if region is None: |
| 391 | return None |
| 392 | return self._make_allocation(region, nbytes, shape, dtype) |
| 393 | |
| 394 | def acquire_cached( |
| 395 | self, mm_hash: str, shape: tuple[int, ...], dtype: torch.dtype |
| 396 | ) -> ResidentLease[MemoryAllocation] | None: |
| 397 | with self.lock: |
| 398 | allocation = self._residents.get(mm_hash) |
| 399 | if allocation is None: |
| 400 | return None |
| 401 | if ( |
| 402 | tuple(allocation.tensor.shape) != shape |
| 403 | or allocation.tensor.dtype != dtype |
| 404 | ): |
| 405 | raise ValueError("conflicting cached tensor for mm_hash") |
| 406 | return self._residents.acquire(mm_hash) |
| 407 | |
| 408 | def take_resident( |
| 409 | self, mm_hash: str, shape: tuple[int, ...], dtype_name: str |
| 410 | ) -> torch.Tensor | None: |
| 411 | with self.lock: |
| 412 | allocation = self._residents.get(mm_hash) |
| 413 | if allocation is None: |
| 414 | return None |
| 415 | tensor = allocation.tensor |
| 416 | if ( |
| 417 | tuple(tensor.shape) != shape |
| 418 | or str(tensor.dtype).split(".")[-1] != dtype_name |
| 419 | ): |
| 420 | return None |
| 421 | self._residents.pin(mm_hash) |
| 422 | self._retire_events.pop(mm_hash, None) |
| 423 | return tensor |
| 424 | |
| 425 | def _record_release_event(self) -> torch.Event | None: |
| 426 | pool = self.tensor |
| 427 | if pool is None or pool.device.type != "cuda": |
| 428 | return None |
| 429 | event = torch.Event() |
| 430 | event.record(torch.accelerator.current_stream(pool.device)) |
| 431 | return event |
| 432 | |
| 433 | def release_cached(self, lease: ResidentLease[MemoryAllocation]) -> None: |
| 434 | with self.lock: |
| 435 | released = self._residents.release(lease) |
| 436 | if released is not None: |
| 437 | self._defer_or_free(released, self._record_release_event()) |
| 438 | |
| 439 | def publish( |
| 440 | self, |
| 441 | mm_hash: str, |
| 442 | allocation: MemoryAllocation, |
| 443 | lease: ResidentLease[MemoryAllocation] | None = None, |
| 444 | *, |
| 445 | pin: bool = True, |
| 446 | ) -> MemoryAllocation: |
| 447 | with self.lock: |
| 448 | if lease is not None: |
| 449 | canonical, released = self._residents.consume(lease) |
| 450 | self._retire_events.pop(mm_hash, None) |
| 451 | if released is not None: |
| 452 | self._defer_or_free(released, self._record_release_event()) |
| 453 | return canonical |
| 454 | previous = self._residents.get(mm_hash) |
| 455 | displaced = self._residents.insert(mm_hash, allocation) |
| 456 | if not pin: |
| 457 | self._residents.retire(mm_hash) |
| 458 | event = None |
| 459 | if previous is not None and previous is not allocation: |
| 460 | event = self._retire_events.pop(mm_hash, None) |
| 461 | if displaced is None or displaced is allocation: |
| 462 | return allocation |
| 463 | if event is None: |
| 464 | event = self._record_release_event() |
| 465 | self._defer_or_free(displaced, event) |
| 466 | return allocation |
| 467 | |
| 468 | def retire_stale( |
| 469 | self, |
| 470 | encoder_cache: dict[str, torch.Tensor], |
| 471 | reserved_hashes: set[str] | None = None, |
| 472 | *, |
| 473 | freed: list[str] | None = None, |
| 474 | ) -> None: |
| 475 | if self.tensor is None: |
| 476 | return |
| 477 | with self.lock: |
| 478 | for mm_hash in self._residents.referenced() if freed is None else freed: |
| 479 | allocation = self._residents.get(mm_hash) |
| 480 | if allocation is None: |
| 481 | continue |
| 482 | if encoder_cache.get(mm_hash) is allocation.tensor: |
| 483 | continue |
| 484 | if reserved_hashes and mm_hash in reserved_hashes: |
| 485 | continue |
| 486 | event = self._record_release_event() |
| 487 | if event is not None: |
| 488 | self._retire_events[mm_hash] = event |
| 489 | self._residents.retire(mm_hash) |
| 490 | self._poll_frees_locked() |
| 491 | |
| 492 | def drain_reclaimed(self) -> set[str]: |
| 493 | with self.lock: |
| 494 | reclaimed = self._reclaimed |
| 495 | self._reclaimed = set() |
| 496 | return reclaimed |
| 497 | |
| 498 | def close(self) -> None: |
| 499 | with self.lock: |
| 500 | if not self._allocator.close(self._transfer): |
| 501 | return |
| 502 | self._residents.clear() |
| 503 | self._retire_events.clear() |
| 504 | self._pending_frees.clear() |