[KV-Offloading] : Expose CPU cache usage metric (#45737)
Signed-off-by: Varun Sundar Rabindranath <varun-sundar-rabindranath@h100-01.nemg-001.lab.rdu2.dc.redhat.com> Signed-off-by: <> Co-authored-by: Varun Sundar Rabindranath <varun-sundar-rabindranath@h100-01.nemg-001.lab.rdu2.dc.redhat.com>
Varun Sundar Rabindranath committed
Jun 20, 2026 at 17:21 UTC
3b4a76b63fb1a6bbf8641fa87f4ecbc9a229ac94
4 files changed
+86
-22
tests/v1/kv_offload/cpu/test_manager.py
+41
-5
@@ -14,12 +14,13 @@ from vllm.v1.kv_offload.base import (
14
ReqContext,
15
make_offload_key,
16
)
17
-from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
17
+from vllm.v1.kv_offload.cpu.common import (
18
+ CPULoadStoreSpec,
19
+ CPUOffloadingMetrics,
20
+)
21
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
22
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
23
21
-STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
22
-
24
25
def make_req_context(
26
req_id: str = "", kv_transfer_params: dict | None = None
@@ -181,10 +182,45 @@ def test_filter_reused_manager_reports_stores_skipped_counter():
182
)
183
stats = manager.get_stats()
184
assert stats is not None
184
- assert stats.reduce()[STORES_SKIPPED] == 3
185
+ assert stats.reduce()[CPUOffloadingMetrics.STORES_SKIPPED] == 3
186
stats = manager.get_stats()
187
assert stats is not None
187
- assert stats.reduce()[STORES_SKIPPED] == 0
188
+ assert stats.reduce()[CPUOffloadingMetrics.STORES_SKIPPED] == 0
189
+
190
+
191
+def test_cpu_manager_reports_cache_usage_gauge():
192
+ def check_usage_stats(manager: CPUOffloadingManager, value: float):
193
+ stats = manager.get_stats()
194
+ assert stats is not None
195
+ assert stats.reduce()[
196
+ CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC
197
+ ] == pytest.approx(value)
198
+
199
+ # Zero-capacity manager always reports 0.0
200
+ manager = make_cpu_manager(num_blocks=0)
201
+ check_usage_stats(manager, 0.0)
202
+
203
+ # Empty manager (4 blocks, none allocated): usage = 0.0
204
+ manager = make_cpu_manager(num_blocks=4)
205
+ check_usage_stats(manager, 0.0)
206
+
207
+ # After allocating 2 of 4 blocks: usage = 0.5
208
+ manager.prepare_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
209
+ check_usage_stats(manager, 0.5)
210
+
211
+ # After filling all 4 blocks: usage = 1.0
212
+ manager.prepare_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
213
+ check_usage_stats(manager, 1.0)
214
+
215
+ # After completing store, the blocks becomes evictable as it is not actively used
216
+ # and usage drops.
217
+ manager.complete_store(to_keys([1, 2]), _EMPTY_REQ_CTX)
218
+ check_usage_stats(manager, 0.5)
219
+
220
+ # After completing store, the blocks becomes evictable as it is not actively used
221
+ # and usage drops.
222
+ manager.complete_store(to_keys([3, 4]), _EMPTY_REQ_CTX)
223
+ check_usage_stats(manager, 0.0)
224
225
226
def test_cpu_manager():
vllm/v1/kv_offload/cpu/common.py
+4
-1
@@ -4,7 +4,10 @@ from typing_extensions import override
4
5
from vllm.v1.kv_offload.base import BlockIDsLoadStoreSpec
6
7
-METRIC_STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
7
+
8
+class CPUOffloadingMetrics:
9
+ STORES_SKIPPED = "vllm:kv_offload_stores_skipped"
10
+ CPU_CACHE_USAGE_PERC = "vllm:kv_offload_cpu_cache_usage_perc"
11
12
13
class CPULoadStoreSpec(BlockIDsLoadStoreSpec):
vllm/v1/kv_offload/cpu/manager.py
+19
-8
@@ -18,7 +18,10 @@ from vllm.v1.kv_offload.base import (
18
ReqContext,
19
RequestOffloadingContext,
20
)
21
-from vllm.v1.kv_offload.cpu.common import METRIC_STORES_SKIPPED, CPULoadStoreSpec
21
+from vllm.v1.kv_offload.cpu.common import (
22
+ CPULoadStoreSpec,
23
+ CPUOffloadingMetrics,
24
+)
25
from vllm.v1.kv_offload.cpu.policies.arc import ARCCachePolicy
26
from vllm.v1.kv_offload.cpu.policies.base import BlockStatus, CachePolicy
27
from vllm.v1.kv_offload.cpu.policies.lru import LRUCachePolicy
@@ -282,13 +285,21 @@ class CPUOffloadingManager(OffloadingManager):
285
self.events.clear()
286
287
def get_stats(self) -> OffloadingConnectorStats | None:
285
- if self.store_threshold < 2:
286
- return None
287
-
288
stats = OffloadingConnectorStats()
289
- stats.increase_counter(
290
- METRIC_STORES_SKIPPED,
291
- self.stores_skipped_in_current_batch,
289
+
290
+ # Compute cache usage.
291
+ num_used = (
292
+ self._num_allocated_blocks
293
+ - len(self._free_list)
294
+ - self._num_evictable_cache_blocks
295
)
293
- self.stores_skipped_in_current_batch = 0
296
+ usage = num_used / self._num_blocks if self._num_blocks > 0 else 0.0
297
+ stats.set_gauge(CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC, usage)
298
+
299
+ if self.store_threshold >= 2:
300
+ stats.increase_counter(
301
+ CPUOffloadingMetrics.STORES_SKIPPED,
302
+ self.stores_skipped_in_current_batch,
303
+ )
304
+ self.stores_skipped_in_current_batch = 0
305
return stats
vllm/v1/kv_offload/cpu/spec.py
+22
-8
@@ -14,11 +14,15 @@ from vllm.v1.kv_offload.base import (
14
GPULoadStoreSpec,
15
LoadStoreSpec,
16
OffloadingCounterMetadata,
17
+ OffloadingGaugeMetadata,
18
OffloadingManager,
19
OffloadingMetricMetadata,
20
OffloadingSpec,
21
)
21
-from vllm.v1.kv_offload.cpu.common import METRIC_STORES_SKIPPED, CPULoadStoreSpec
22
+from vllm.v1.kv_offload.cpu.common import (
23
+ CPULoadStoreSpec,
24
+ CPUOffloadingMetrics,
25
+)
26
from vllm.v1.kv_offload.cpu.gpu_worker import CpuGpuOffloadingHandlers
27
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
28
from vllm.v1.kv_offload.worker.worker import OffloadingHandler
@@ -31,17 +35,27 @@ class CPUOffloadingSpec(OffloadingSpec):
35
def build_metric_definitions(
36
cls, extra_config: dict[str, Any]
37
) -> dict[str, OffloadingMetricMetadata]:
34
- store_threshold = int(extra_config.get("store_threshold", 0))
35
- if store_threshold < 2:
36
- return {}
37
- return {
38
- METRIC_STORES_SKIPPED: OffloadingCounterMetadata(
38
+ definitions: dict[str, OffloadingMetricMetadata] = {
39
+ CPUOffloadingMetrics.CPU_CACHE_USAGE_PERC: OffloadingGaugeMetadata(
40
documentation=(
40
- "Number of KV offload stores skipped because the reuse "
41
- "threshold was not reached."
41
+ "Fraction of CPU KV-cache space currently pinned by active "
42
+ "transfers (0.0 = idle, 1.0 = saturated). Sustained high "
43
+ "values indicate transfers (stores or promotions) may be "
44
+ "dropped due to insufficient capacity."
45
),
46
)
47
}
48
+ store_threshold = int(extra_config.get("store_threshold", 0))
49
+ if store_threshold >= 2:
50
+ definitions[CPUOffloadingMetrics.STORES_SKIPPED] = (
51
+ OffloadingCounterMetadata(
52
+ documentation=(
53
+ "Number of KV offload stores skipped because the reuse "
54
+ "threshold was not reached."
55
+ ),
56
+ )
57
+ )
58
+ return definitions
59
60
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig):
61
super().__init__(vllm_config, kv_cache_config)