| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """ExampleSecondaryTierManager: A simple in-memory secondary tier. |
| 4 | |
| 5 | This implementation provides a minimal secondary tier that stores chunks |
| 6 | in memory (using a dictionary) with immediate completion. It serves as a |
| 7 | reference for writing new tiers and is useful for testing the |
| 8 | TieringOffloadingManager without requiring actual storage or network backends. |
| 9 | """ |
| 10 | |
| 11 | import logging |
| 12 | from collections.abc import Iterable |
| 13 | from typing import TYPE_CHECKING, ClassVar |
| 14 | |
| 15 | from typing_extensions import override |
| 16 | |
| 17 | from vllm.v1.kv_offload.base import ( |
| 18 | LookupResult, |
| 19 | Medium, |
| 20 | OffloadKey, |
| 21 | ReqContext, |
| 22 | RequestOffloadingContext, |
| 23 | ) |
| 24 | from vllm.v1.kv_offload.tiering.base import ( |
| 25 | JobResult, |
| 26 | SecondaryTierManager, |
| 27 | TransferJob, |
| 28 | ) |
| 29 | |
| 30 | logger = logging.getLogger(__name__) |
| 31 | |
| 32 | if TYPE_CHECKING: |
| 33 | from vllm.v1.kv_offload.base import OffloadingSpec |
| 34 | from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector |
| 35 | |
| 36 | |
| 37 | class ExampleSecondaryTierManager(SecondaryTierManager): |
| 38 | """A simple in-memory secondary tier. |
| 39 | |
| 40 | This implementation: |
| 41 | - Stores chunks in a dictionary (key -> True) |
| 42 | - Completes transfers immediately (synchronous) |
| 43 | """ |
| 44 | |
| 45 | medium: ClassVar[Medium] = Medium.CPU |
| 46 | |
| 47 | def __init__( |
| 48 | self, |
| 49 | offloading_spec: "OffloadingSpec", |
| 50 | primary_kv_view: memoryview, |
| 51 | tier_type: str, |
| 52 | custom_param: int = 0, |
| 53 | backpressure_detector: "BackpressureDetector | None" = None, |
| 54 | ): |
| 55 | """Initialize the example secondary tier. |
| 56 | |
| 57 | Args: |
| 58 | offloading_spec: The offloading spec this tier belongs to. |
| 59 | primary_kv_view: Memoryview over the primary tier's KV buffer. |
| 60 | tier_type: Name identifying this tier type. |
| 61 | custom_param: Dummy parameter demonstrating custom args. |
| 62 | backpressure_detector: Optional backpressure detector. |
| 63 | |
| 64 | """ |
| 65 | super().__init__( |
| 66 | offloading_spec=offloading_spec, |
| 67 | primary_kv_view=primary_kv_view, |
| 68 | tier_type=tier_type, |
| 69 | backpressure_detector=backpressure_detector, |
| 70 | ) |
| 71 | |
| 72 | logger.info( |
| 73 | "ExampleSecondaryTierManager initialized with custom_param=%d", custom_param |
| 74 | ) |
| 75 | |
| 76 | # key -> True (only care about presence) |
| 77 | self.chunks: dict[OffloadKey, bool] = {} |
| 78 | |
| 79 | # Completed jobs waiting to be retrieved by get_finished_jobs() |
| 80 | self.completed_jobs: list[JobResult] = [] |
| 81 | assert primary_kv_view.strides is not None |
| 82 | self._chunk_size = primary_kv_view.strides[0] |
| 83 | |
| 84 | @override |
| 85 | def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: |
| 86 | """Check whether a chunk exists in this secondary tier. |
| 87 | |
| 88 | Args: |
| 89 | key: Offload key to look up. |
| 90 | req_context: Per-request context. |
| 91 | |
| 92 | Returns: |
| 93 | HIT if the chunk is present, MISS if not found. |
| 94 | |
| 95 | """ |
| 96 | return LookupResult.HIT if key in self.chunks else LookupResult.MISS |
| 97 | |
| 98 | @override |
| 99 | def submit_store(self, job_metadata: TransferJob) -> None: |
| 100 | """Submit a job to store chunks from primary tier to this tier. |
| 101 | |
| 102 | Args: |
| 103 | job_metadata: Job metadata including job_id, keys, and |
| 104 | spec for reading chunks from the primary tier. |
| 105 | |
| 106 | """ |
| 107 | keys = job_metadata.keys |
| 108 | chunk_ids = job_metadata.chunk_ids |
| 109 | |
| 110 | assert len(keys) == len(chunk_ids), ( |
| 111 | f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids" |
| 112 | ) |
| 113 | |
| 114 | for key in keys: |
| 115 | self.chunks[key] = True |
| 116 | self.completed_jobs.append( |
| 117 | JobResult( |
| 118 | job_id=job_metadata.job_id, |
| 119 | success=True, |
| 120 | transfer_time=0.0, |
| 121 | ) |
| 122 | ) |
| 123 | |
| 124 | @override |
| 125 | def submit_load(self, job_metadata: TransferJob) -> None: |
| 126 | """Submit a job to load chunks from this tier to primary tier. |
| 127 | |
| 128 | Args: |
| 129 | job_metadata: Job metadata including job_id, keys, and |
| 130 | spec for writing chunks into the primary tier. |
| 131 | |
| 132 | """ |
| 133 | keys = job_metadata.keys |
| 134 | chunk_ids = job_metadata.chunk_ids |
| 135 | |
| 136 | assert len(keys) == len(chunk_ids), ( |
| 137 | f"Length mismatch: {len(keys)} keys but {len(chunk_ids)} chunk_ids" |
| 138 | ) |
| 139 | |
| 140 | for key in keys: |
| 141 | if key not in self.chunks: |
| 142 | self.completed_jobs.append( |
| 143 | JobResult(job_id=job_metadata.job_id, success=False) |
| 144 | ) |
| 145 | return |
| 146 | |
| 147 | self.completed_jobs.append( |
| 148 | JobResult( |
| 149 | job_id=job_metadata.job_id, |
| 150 | success=True, |
| 151 | transfer_time=0.0, |
| 152 | ) |
| 153 | ) |
| 154 | |
| 155 | @override |
| 156 | def get_finished_jobs(self) -> Iterable[JobResult]: |
| 157 | """Poll for finished jobs. |
| 158 | |
| 159 | Returns: |
| 160 | Iterable of JobResult objects for all jobs that have |
| 161 | finished since the last call. |
| 162 | |
| 163 | """ |
| 164 | result = self.completed_jobs |
| 165 | self.completed_jobs = [] |
| 166 | return result |
| 167 | |
| 168 | @override |
| 169 | def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: |
| 170 | return RequestOffloadingContext() |
| 171 | |
| 172 | @override |
| 173 | def drain_jobs(self) -> None: |
| 174 | """Synchronous tier — submit_*() returns only after the operation |
| 175 | completes, so there is nothing to wait for.""" |
| 176 | return |
| 177 | |
| 178 | def get_num_chunks(self) -> int: |
| 179 | """Get the number of chunks currently stored in this tier.""" |
| 180 | return len(self.chunks) |