| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Self-describing KV cache events for the offloading connector. |
| 4 | |
| 5 | The OffloadingManager identifies an offloaded chunk only by its OffloadKey, |
| 6 | so its raw events carry no token ids, parent hash, or block size. |
| 7 | :class:`OffloadingEventsTracker` snapshots each chunk's full ``BlockStored`` |
| 8 | payload while the ``Request`` is alive and publishes stores as block-granular |
| 9 | payloads: a chunk event may carry multiple constituent per-block hashes, and |
| 10 | evictions fan out to the same hashes. Chunks overlapping a non-chunk-aligned |
| 11 | shared prefix re-announce the shared hashes once per chunk; consumers are |
| 12 | expected to deduplicate (reference-count) repeated store/remove announcements |
| 13 | of the same hash. Opt-in via |
| 14 | ``kv_connector_extra_config["self_describing_kv_events"]``; inert unless |
| 15 | KV cache events are enabled. See the PR description for the full design. |
| 16 | """ |
| 17 | |
| 18 | from collections.abc import Iterable |
| 19 | from dataclasses import dataclass |
| 20 | from typing import TYPE_CHECKING, Any, NamedTuple |
| 21 | |
| 22 | from vllm.distributed.kv_events import ( |
| 23 | MEDIUM_CPU, |
| 24 | MEDIUM_STORAGE, |
| 25 | BlockRemoved, |
| 26 | BlockStored, |
| 27 | KVCacheEvent, |
| 28 | ) |
| 29 | from vllm.logger import init_logger |
| 30 | from vllm.v1.core.kv_cache_utils import ( |
| 31 | BlockHash, |
| 32 | maybe_convert_block_hash, |
| 33 | resolve_block_hashes, |
| 34 | to_event_extra_keys, |
| 35 | ) |
| 36 | from vllm.v1.kv_cache_interface import ( |
| 37 | KVCacheGroupSpec, |
| 38 | get_kv_cache_spec_kind, |
| 39 | get_kv_cache_spec_sliding_window, |
| 40 | ) |
| 41 | from vllm.v1.kv_offload.base import ( |
| 42 | Medium, |
| 43 | OffloadingEvent, |
| 44 | OffloadingKVEventsConfig, |
| 45 | OffloadKey, |
| 46 | get_offload_block_hash, |
| 47 | get_offload_group_idx, |
| 48 | ) |
| 49 | from vllm.v1.request import Request |
| 50 | |
| 51 | if TYPE_CHECKING: |
| 52 | from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import ( |
| 53 | GroupOffloadConfig, |
| 54 | ) |
| 55 | |
| 56 | logger = init_logger(__name__) |
| 57 | |
| 58 | _MEDIUM_TO_EVENT_STR: dict[Medium, str] = { |
| 59 | Medium.CPU: MEDIUM_CPU, |
| 60 | Medium.STORAGE: MEDIUM_STORAGE, |
| 61 | } |
| 62 | |
| 63 | |
| 64 | class OffloadingEventGroupSpec(NamedTuple): |
| 65 | kv_cache_spec_kind: str | None |
| 66 | kv_cache_spec_sliding_window: int | None |
| 67 | |
| 68 | |
| 69 | def get_offloading_event_group_spec( |
| 70 | kv_cache_group: KVCacheGroupSpec, |
| 71 | ) -> OffloadingEventGroupSpec: |
| 72 | kv_cache_spec = kv_cache_group.kv_cache_spec |
| 73 | return OffloadingEventGroupSpec( |
| 74 | kv_cache_spec_kind=get_kv_cache_spec_kind(kv_cache_spec).value, |
| 75 | kv_cache_spec_sliding_window=get_kv_cache_spec_sliding_window(kv_cache_spec), |
| 76 | ) |
| 77 | |
| 78 | |
| 79 | @dataclass(slots=True) |
| 80 | class _OffloadEventMetadata: |
| 81 | """BlockStored payload snapshot for one OffloadKey, captured while the |
| 82 | Request is available and kept until the final matching removal event. |
| 83 | ``medium`` and ``ownership`` are forwarded from the OffloadingEvent.""" |
| 84 | |
| 85 | # The chunk's constituent block hashes; the last one is the OffloadKey. |
| 86 | block_hashes: tuple[BlockHash, ...] |
| 87 | parent_block_hash: BlockHash | None |
| 88 | token_ids: tuple[int, ...] |
| 89 | block_size: int |
| 90 | lora_id: int | None |
| 91 | lora_name: str | None |
| 92 | # Deferred: needs the same incremental curr_mm_idx handling as GPU events. |
| 93 | extra_keys: tuple[tuple[Any, ...] | None, ...] | None |
| 94 | group_idx: int |
| 95 | kv_cache_spec: OffloadingEventGroupSpec |
| 96 | active_residencies: set[tuple[Medium, str | None]] |
| 97 | |
| 98 | |
| 99 | class OffloadingEventsTracker: |
| 100 | """Tracks offloaded chunks' KV event payloads from store to eviction. |
| 101 | |
| 102 | The scheduler calls :meth:`record_store` from ``_build_store_jobs`` and |
| 103 | :meth:`record_lookup` for ready primary-tier hits while the ``Request`` is |
| 104 | available. Deferred and missing lookups add no state. Under the connector's |
| 105 | supported success-only transfer model, entries remain until the final |
| 106 | observed residency removal or :meth:`reset`. |
| 107 | """ |
| 108 | |
| 109 | def __init__(self, config: OffloadingKVEventsConfig): |
| 110 | self.config = config |
| 111 | self.self_describing_enabled = ( |
| 112 | config.enable_kv_cache_events and config.self_describing_kv_events |
| 113 | ) |
| 114 | |
| 115 | # OffloadKey -> payload snapshot, kept until final removal or reset. |
| 116 | self._pending_event_metadata: dict[OffloadKey, _OffloadEventMetadata] = {} |
| 117 | |
| 118 | def record_store( |
| 119 | self, |
| 120 | req: Request, |
| 121 | group_config: "GroupOffloadConfig", |
| 122 | chunk_idx: int, |
| 123 | offload_key: OffloadKey, |
| 124 | ) -> None: |
| 125 | """Snapshot the KV cache event payload for one offloaded chunk. |
| 126 | |
| 127 | No-op when self-describing event capture is disabled or for |
| 128 | sliding-window / SSM groups, which keep the legacy placeholder payload. |
| 129 | """ |
| 130 | if not self.self_describing_enabled: |
| 131 | return |
| 132 | if group_config.sliding_window_size_in_chunks is not None: |
| 133 | return |
| 134 | meta = self._build_event_metadata(req, group_config, chunk_idx) |
| 135 | if existing := self._pending_event_metadata.get(offload_key): |
| 136 | meta.active_residencies.update(existing.active_residencies) |
| 137 | self._pending_event_metadata[offload_key] = meta |
| 138 | |
| 139 | def record_lookup( |
| 140 | self, |
| 141 | req: Request, |
| 142 | group_config: "GroupOffloadConfig", |
| 143 | chunk_idx: int, |
| 144 | offload_key: OffloadKey, |
| 145 | ) -> None: |
| 146 | """Snapshot metadata for a ready primary-tier lookup hit.""" |
| 147 | if not self.self_describing_enabled: |
| 148 | return |
| 149 | if group_config.sliding_window_size_in_chunks is not None: |
| 150 | return |
| 151 | if offload_key not in self._pending_event_metadata: |
| 152 | self._pending_event_metadata[offload_key] = self._build_event_metadata( |
| 153 | req, group_config, chunk_idx |
| 154 | ) |
| 155 | |
| 156 | def record_partial_store( |
| 157 | self, |
| 158 | req: Request, |
| 159 | group_config: "GroupOffloadConfig", |
| 160 | boundary_tokens: int, |
| 161 | offload_key: OffloadKey, |
| 162 | ) -> None: |
| 163 | """Snapshot metadata for a newly stored partial recurrent tail.""" |
| 164 | if group_config.sliding_window_size_in_chunks is not None: |
| 165 | return |
| 166 | self._record_partial_tail(req, group_config, boundary_tokens, offload_key) |
| 167 | |
| 168 | def record_partial_lookup( |
| 169 | self, |
| 170 | req: Request, |
| 171 | group_config: "GroupOffloadConfig", |
| 172 | boundary_tokens: int, |
| 173 | offload_key: OffloadKey, |
| 174 | ) -> None: |
| 175 | """Backfill metadata for a partial recurrent tail lookup hit.""" |
| 176 | if group_config.sliding_window_size_in_chunks is not None: |
| 177 | return |
| 178 | if offload_key not in self._pending_event_metadata: |
| 179 | self._record_partial_tail(req, group_config, boundary_tokens, offload_key) |
| 180 | |
| 181 | def _record_partial_tail( |
| 182 | self, |
| 183 | req: Request, |
| 184 | group_config: "GroupOffloadConfig", |
| 185 | boundary_tokens: int, |
| 186 | offload_key: OffloadKey, |
| 187 | ) -> None: |
| 188 | """Build metadata for the valid prefix of one physical cache block. |
| 189 | |
| 190 | A partial recurrent tail ends on a hash boundary but before its |
| 191 | physical cache block is full. The event describes only the valid |
| 192 | hashes and tokens, not the unused remainder of that physical block. |
| 193 | """ |
| 194 | if not self.self_describing_enabled: |
| 195 | return |
| 196 | |
| 197 | tokens_per_hash = group_config.tokens_per_chunk // group_config.hashes_per_chunk |
| 198 | # Subtract one so the boundary token itself cannot select the next |
| 199 | # physical block when the boundary lies exactly on a block edge. |
| 200 | chunk_start = ( |
| 201 | (boundary_tokens - 1) // group_config.tokens_per_chunk |
| 202 | ) * group_config.tokens_per_chunk |
| 203 | first_hash_idx = chunk_start // tokens_per_hash |
| 204 | last_hash_idx = boundary_tokens // tokens_per_hash |
| 205 | assert chunk_start < boundary_tokens |
| 206 | assert boundary_tokens % tokens_per_hash == 0 |
| 207 | assert last_hash_idx <= len(req.block_hashes) |
| 208 | |
| 209 | # Unlike a complete chunk, a partial tail contains only the hashes |
| 210 | # between its physical block start and its valid token boundary. |
| 211 | maybe_block_hashes = req.block_hashes[first_hash_idx:last_hash_idx] |
| 212 | block_hashes = tuple( |
| 213 | block_hash for block_hash in maybe_block_hashes if block_hash is not None |
| 214 | ) |
| 215 | assert block_hashes and len(block_hashes) == len(maybe_block_hashes) |
| 216 | parent_block_hash = ( |
| 217 | req.block_hashes[first_hash_idx - 1] if first_hash_idx > 0 else None |
| 218 | ) |
| 219 | assert first_hash_idx == 0 or parent_block_hash is not None |
| 220 | |
| 221 | lora_id = req.lora_request.adapter_id if req.lora_request is not None else None |
| 222 | lora_name = req.lora_request.name if req.lora_request is not None else None |
| 223 | meta = _OffloadEventMetadata( |
| 224 | block_hashes=block_hashes, |
| 225 | parent_block_hash=parent_block_hash, |
| 226 | token_ids=tuple(req.all_token_ids[chunk_start:boundary_tokens]), |
| 227 | block_size=tokens_per_hash, |
| 228 | lora_id=lora_id, |
| 229 | lora_name=lora_name, |
| 230 | extra_keys=None, |
| 231 | group_idx=group_config.group_idx, |
| 232 | kv_cache_spec=group_config.kv_event_group_spec, |
| 233 | active_residencies={(Medium.CPU, None)}, |
| 234 | ) |
| 235 | if existing := self._pending_event_metadata.get(offload_key): |
| 236 | meta.active_residencies.update(existing.active_residencies) |
| 237 | self._pending_event_metadata[offload_key] = meta |
| 238 | |
| 239 | def take_events(self, events: Iterable[OffloadingEvent]) -> Iterable[KVCacheEvent]: |
| 240 | """Translate raw OffloadingEvents into self-describing KV events. |
| 241 | |
| 242 | Complete metadata is available only for full-attention groups when |
| 243 | the tracker is enabled. Other shapes retain the legacy placeholder |
| 244 | payload so consumers can ignore them. |
| 245 | |
| 246 | Yields: |
| 247 | ``BlockStored`` or ``BlockRemoved`` events corresponding to |
| 248 | the underlying :class:`OffloadingEvent` stream. |
| 249 | |
| 250 | """ |
| 251 | removed_keys: set[OffloadKey] = set() |
| 252 | for event in events: |
| 253 | if event.removed: |
| 254 | if self.self_describing_enabled: |
| 255 | removed_keys.update(event.keys) |
| 256 | yield from self._take_removed_event(event) |
| 257 | else: |
| 258 | yield from self._take_stored_event(event) |
| 259 | |
| 260 | # A primary removal can precede a queued secondary store in this batch. |
| 261 | # Keep its payload until all stores have registered their residencies. |
| 262 | for key in removed_keys: |
| 263 | meta = self._pending_event_metadata.get(key) |
| 264 | if meta is not None and not meta.active_residencies: |
| 265 | self._pending_event_metadata.pop(key) |
| 266 | |
| 267 | def reset(self) -> None: |
| 268 | """Drop all tracked state; pending payloads are stale after a |
| 269 | manager cache reset.""" |
| 270 | self._pending_event_metadata.clear() |
| 271 | |
| 272 | def _build_event_metadata( |
| 273 | self, |
| 274 | req: Request, |
| 275 | group_config: "GroupOffloadConfig", |
| 276 | chunk_idx: int, |
| 277 | ) -> _OffloadEventMetadata: |
| 278 | """Build the payload snapshot for one offloaded chunk: its |
| 279 | constituent per-block hashes, the whole chunk's tokens, and the |
| 280 | per-block ``block_size``.""" |
| 281 | hashes_per_chunk = group_config.hashes_per_chunk |
| 282 | assert hashes_per_chunk > 0 |
| 283 | assert chunk_idx >= 0 |
| 284 | tokens_per_hash = group_config.tokens_per_chunk // hashes_per_chunk |
| 285 | # Each chunk's final raw hash is its OffloadKey. |
| 286 | first_hash_idx = chunk_idx * hashes_per_chunk |
| 287 | last_hash_idx = first_hash_idx + hashes_per_chunk |
| 288 | assert first_hash_idx >= 0 |
| 289 | assert last_hash_idx <= len(req.block_hashes) |
| 290 | raw_chunk_hashes = req.block_hashes[first_hash_idx:last_hash_idx] |
| 291 | chunk_hashes = resolve_block_hashes( |
| 292 | raw_chunk_hashes, |
| 293 | tokens_per_hash, |
| 294 | group_config.tokens_per_block, |
| 295 | ) |
| 296 | for block_hash in chunk_hashes: |
| 297 | assert block_hash is not None |
| 298 | assert len(chunk_hashes) == ( |
| 299 | group_config.tokens_per_chunk // group_config.tokens_per_block |
| 300 | ) |
| 301 | |
| 302 | if group_config.sliding_window_size_in_chunks is not None: |
| 303 | # The recording methods filter these out before calling this helper. |
| 304 | raise AssertionError("self-describing events only support full attention") |
| 305 | |
| 306 | parent_block_hash: BlockHash | None |
| 307 | if first_hash_idx == 0: |
| 308 | parent_block_hash = None |
| 309 | else: |
| 310 | parent_block_hash = req.block_hashes[first_hash_idx - 1] |
| 311 | assert parent_block_hash is not None |
| 312 | |
| 313 | tok_start = chunk_idx * group_config.tokens_per_chunk |
| 314 | tok_end = tok_start + group_config.tokens_per_chunk |
| 315 | assert tok_end <= len(req.all_token_ids) |
| 316 | token_ids = tuple(req.all_token_ids[tok_start:tok_end]) |
| 317 | |
| 318 | lora_id: int | None = None |
| 319 | lora_name: str | None = None |
| 320 | if req.lora_request is not None: |
| 321 | lora_id = req.lora_request.adapter_id |
| 322 | lora_name = req.lora_request.name |
| 323 | |
| 324 | return _OffloadEventMetadata( |
| 325 | block_hashes=tuple(chunk_hashes), |
| 326 | parent_block_hash=parent_block_hash, |
| 327 | token_ids=token_ids, |
| 328 | block_size=group_config.tokens_per_block, |
| 329 | lora_id=lora_id, |
| 330 | lora_name=lora_name, |
| 331 | extra_keys=None, |
| 332 | group_idx=group_config.group_idx, |
| 333 | kv_cache_spec=group_config.kv_event_group_spec, |
| 334 | active_residencies={(Medium.CPU, None)}, |
| 335 | ) |
| 336 | |
| 337 | def _placeholder_stored( |
| 338 | self, |
| 339 | key: OffloadKey, |
| 340 | medium: Medium, |
| 341 | locality: str | None, |
| 342 | ownership: str | None, |
| 343 | ) -> BlockStored: |
| 344 | return BlockStored( |
| 345 | block_hashes=[ |
| 346 | maybe_convert_block_hash(BlockHash(get_offload_block_hash(key))) |
| 347 | ], |
| 348 | parent_block_hash=None, |
| 349 | token_ids=[], |
| 350 | lora_id=None, |
| 351 | block_size=0, |
| 352 | medium=_MEDIUM_TO_EVENT_STR[medium], |
| 353 | lora_name=None, |
| 354 | group_idx=get_offload_group_idx(key), |
| 355 | locality=locality, |
| 356 | ownership=ownership, |
| 357 | ) |
| 358 | |
| 359 | def _take_stored_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]: |
| 360 | # Metadata is read, NOT popped: the entry must survive until the |
| 361 | # eviction event so BlockRemoved can fan out to the same hashes. |
| 362 | # Events are self-contained (own parent), so key order is free. |
| 363 | locality = event.locality.value if event.locality is not None else None |
| 364 | for key in event.keys: |
| 365 | meta = self._pending_event_metadata.get(key) |
| 366 | if meta is None: |
| 367 | if self.self_describing_enabled: |
| 368 | # Expected for unsupported shapes; warn once only. |
| 369 | logger.warning_once( |
| 370 | "OffloadingEventsTracker: no event metadata for " |
| 371 | "offload key during BlockStored emission; emitting a " |
| 372 | "placeholder payload. Expected for non-full-attention " |
| 373 | "groups and promotions not observed as a primary-tier " |
| 374 | "hit before translation." |
| 375 | ) |
| 376 | yield self._placeholder_stored( |
| 377 | key, event.medium, locality, event.ownership |
| 378 | ) |
| 379 | continue |
| 380 | |
| 381 | if event.removal_expected: |
| 382 | meta.active_residencies.add((event.medium, event.ownership)) |
| 383 | yield BlockStored( |
| 384 | block_hashes=list( |
| 385 | maybe_convert_block_hash(h) for h in meta.block_hashes |
| 386 | ), |
| 387 | parent_block_hash=( |
| 388 | maybe_convert_block_hash(meta.parent_block_hash) |
| 389 | if meta.parent_block_hash is not None |
| 390 | else None |
| 391 | ), |
| 392 | token_ids=list(meta.token_ids), |
| 393 | block_size=meta.block_size, |
| 394 | lora_id=meta.lora_id, |
| 395 | medium=_MEDIUM_TO_EVENT_STR[event.medium], |
| 396 | lora_name=meta.lora_name, |
| 397 | extra_keys=to_event_extra_keys(meta.extra_keys), |
| 398 | group_idx=meta.group_idx, |
| 399 | kv_cache_spec_kind=meta.kv_cache_spec.kv_cache_spec_kind, |
| 400 | kv_cache_spec_sliding_window=( |
| 401 | meta.kv_cache_spec.kv_cache_spec_sliding_window |
| 402 | ), |
| 403 | locality=locality, |
| 404 | ownership=event.ownership, |
| 405 | ) |
| 406 | |
| 407 | def _take_removed_event(self, event: OffloadingEvent) -> Iterable[KVCacheEvent]: |
| 408 | # Keep group_idx unambiguous if a manager batch spans groups. |
| 409 | locality = event.locality.value if event.locality is not None else None |
| 410 | by_group: dict[int, list] = {} |
| 411 | for key in event.keys: |
| 412 | meta = self._pending_event_metadata.get(key) |
| 413 | if meta is not None: |
| 414 | group_idx = meta.group_idx |
| 415 | by_group.setdefault(group_idx, []).extend( |
| 416 | maybe_convert_block_hash(h) for h in meta.block_hashes |
| 417 | ) |
| 418 | meta.active_residencies.discard((event.medium, event.ownership)) |
| 419 | else: |
| 420 | if self.self_describing_enabled: |
| 421 | logger.warning_once( |
| 422 | "OffloadingEventsTracker: no event metadata for " |
| 423 | "offload key during BlockRemoved emission; emitting a " |
| 424 | "placeholder removal. Expected if the matching store " |
| 425 | "used the legacy placeholder payload; otherwise " |
| 426 | "indicates missing store metadata." |
| 427 | ) |
| 428 | group_idx = get_offload_group_idx(key) |
| 429 | by_group.setdefault(group_idx, []).append( |
| 430 | maybe_convert_block_hash(BlockHash(get_offload_block_hash(key))) |
| 431 | ) |
| 432 | |
| 433 | for group_idx, hashes in by_group.items(): |
| 434 | yield BlockRemoved( |
| 435 | block_hashes=hashes, |
| 436 | medium=_MEDIUM_TO_EVENT_STR[event.medium], |
| 437 | group_idx=group_idx, |
| 438 | locality=locality, |
| 439 | ownership=event.ownership, |
| 440 | ) |