A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 06:00 UTC
[Core][Observability] Add KV cache residency metrics (#27793)
Introduces three new Prometheus histograms for fine-grained observability of KV cache residency behavior: vllm:kv_block_lifetime_seconds — total lifetime from allocation to free vllm:kv_block_idle_before_evict_seconds — idle duration before eviction vllm:kv_block_reuse_gap_seconds — time between consecutive reuses of the same block These metrics help operators analyze KV cache efficiency, reuse patterns, and eviction timing beyond simple utilization rates. Implementation uses monotonic timestamps for accuracy, 1% sampling for minimal overhead (~48 bytes/block), and is fully thread-safe with zero runtime cost when disabled. Two new runtime flags are introduced: --kv-cache-metrics – enable KV cache residency metrics --kv-cache-metrics-sample – control sampling ratio (default: 0.01) Signed-off-by: Shivam <shivamprasad91@gmail.com>
shivampr committed
Dec 1, 2025 at 10:27 UTC
cabc77cc86586206134a19e3e70aa401ec30b71f
11 files changed
+534
-13
docs/design/metrics.md
+23
@@ -263,6 +263,29 @@ record:
263
- End-to-end latency - the interval between frontend `arrival_time`
264
and the frontend receiving the final token.
265
266
+### KV Cache Residency Metrics
267
+
268
+We also emit a set of histograms that describe how long sampled KV cache
269
+blocks stay resident and how often they are reused. Sampling
270
+(`--kv-cache-metrics-sample`) keeps the overhead tiny; when a block is
271
+chosen we record:
272
+
273
+- `lifetime` – allocation ⟶ eviction
274
+- `idle before eviction` – last touch ⟶ eviction
275
+- `reuse gaps` – the pauses between touches when the block gets reused
276
+
277
+Those map directly to the Prometheus metrics:
278
+
279
+- `vllm:kv_block_lifetime_seconds` – how long each sampled block exists.
280
+- `vllm:kv_block_idle_before_evict_seconds` – idle tail after the final access.
281
+- `vllm:kv_block_reuse_gap_seconds` – time between consecutive touches.
282
+
283
+The engine core only ships raw eviction events via `SchedulerStats`; the
284
+frontend drains them, turns them into Prometheus observations, and also
285
+exposes the same data through `LLM.get_metrics()` when logging is on.
286
+Looking at lifetime and idle time on one chart makes it easy to spot
287
+stranded cache or workloads that pin prompts for a long decode.
288
+
289
### Metrics Publishing - Logging
290
291
The `LoggingStatLogger` metrics publisher outputs a log `INFO` message
tests/v1/core/test_kv_cache_metrics.py
new
+224
@@ -0,0 +1,224 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from unittest.mock import patch
5
+
6
+import pytest
7
+
8
+from vllm.v1.core.kv_cache_metrics import (
9
+ BlockMetricsState,
10
+ KVCacheMetricsCollector,
11
+)
12
+from vllm.v1.core.kv_cache_utils import KVCacheBlock
13
+
14
+
15
+class TestBlockMetricsState:
16
+ def test_init(self):
17
+ with patch("time.monotonic_ns", return_value=1000000000):
18
+ state = BlockMetricsState()
19
+ assert state.birth_time_ns == 1000000000
20
+ assert state.last_access_ns == 1000000000
21
+ assert len(state.access_history) == 0
22
+
23
+ def test_access_tracking(self):
24
+ with patch("time.monotonic_ns", return_value=1000000000):
25
+ state = BlockMetricsState()
26
+
27
+ with patch("time.monotonic_ns", return_value=2000000000):
28
+ state.record_access()
29
+
30
+ assert state.last_access_ns == 2000000000
31
+ assert list(state.access_history) == [2000000000]
32
+
33
+ def test_ring_buffer_wraps_at_4(self):
34
+ with patch("time.monotonic_ns", return_value=1000000000):
35
+ state = BlockMetricsState()
36
+
37
+ for i in range(5):
38
+ t = 1000000000 + (i + 1) * 1000000000
39
+ with patch("time.monotonic_ns", return_value=t):
40
+ state.record_access()
41
+
42
+ assert len(state.access_history) == 4
43
+ assert list(state.access_history) == [
44
+ 3000000000,
45
+ 4000000000,
46
+ 5000000000,
47
+ 6000000000,
48
+ ]
49
+
50
+ def test_lifetime(self):
51
+ with patch("time.monotonic_ns", return_value=1000000000):
52
+ state = BlockMetricsState()
53
+ with patch("time.monotonic_ns", return_value=6500000000):
54
+ assert abs(state.get_lifetime_seconds() - 5.5) < 0.001
55
+
56
+ def test_idle_time(self):
57
+ with patch("time.monotonic_ns", return_value=1000000000):
58
+ state = BlockMetricsState()
59
+ state.last_access_ns = 2000000000
60
+ with patch("time.monotonic_ns", return_value=5200000000):
61
+ assert abs(state.get_idle_time_seconds() - 3.2) < 0.001
62
+
63
+ def test_reuse_gaps(self):
64
+ with patch("time.monotonic_ns", return_value=1000000000):
65
+ state = BlockMetricsState()
66
+
67
+ base = 1000000000
68
+ for offset in [0, 1.5, 3.0, 5.5]:
69
+ state.access_history.append(base + int(offset * 1e9))
70
+
71
+ gaps = state.get_reuse_gaps_seconds()
72
+ assert len(gaps) == 3
73
+ assert gaps[0] == 1.5 and gaps[1] == 1.5 and gaps[2] == 2.5
74
+
75
+ def test_ring_wrap_only_gives_3_gaps(self):
76
+ # 5 accesses in size-4 buffer = 3 gaps
77
+ with patch("time.monotonic_ns", return_value=1000000000):
78
+ state = BlockMetricsState()
79
+
80
+ for i in range(5):
81
+ state.access_history.append(1000000000 + i * 1000000000)
82
+
83
+ assert len(state.get_reuse_gaps_seconds()) == 3
84
+
85
+
86
+class TestKVCacheMetricsCollector:
87
+ def test_sample_rate_validation(self):
88
+ with pytest.raises(AssertionError):
89
+ KVCacheMetricsCollector(sample_rate=-0.1)
90
+ with pytest.raises(AssertionError):
91
+ KVCacheMetricsCollector(sample_rate=1.5)
92
+ with pytest.raises(AssertionError):
93
+ KVCacheMetricsCollector(sample_rate=0.0)
94
+
95
+ def test_sampling(self):
96
+ c = KVCacheMetricsCollector(sample_rate=1.0)
97
+ assert sum(1 for _ in range(100) if c.should_sample_block()) == 100
98
+
99
+ c = KVCacheMetricsCollector(sample_rate=0.5)
100
+ samples = sum(1 for _ in range(1000) if c.should_sample_block())
101
+ assert 400 < samples < 600
102
+
103
+ def test_alloc(self):
104
+ c = KVCacheMetricsCollector(sample_rate=1.0)
105
+
106
+ blocks = [KVCacheBlock(block_id=i) for i in range(5)]
107
+ with patch("time.monotonic_ns", return_value=1000000000):
108
+ for block in blocks:
109
+ c.on_block_allocated(block)
110
+
111
+ assert len(c.block_metrics) == 5
112
+
113
+ def test_access(self):
114
+ c = KVCacheMetricsCollector(sample_rate=1.0)
115
+ block = KVCacheBlock(block_id=0)
116
+
117
+ with patch("time.monotonic_ns", return_value=1000000000):
118
+ c.on_block_allocated(block)
119
+
120
+ for i in range(3):
121
+ t = 1000000000 + (i + 1) * 1000000000
122
+ with patch("time.monotonic_ns", return_value=t):
123
+ c.on_block_accessed(block)
124
+
125
+ assert len(c.block_metrics[0].access_history) == 3
126
+
127
+ def test_evict_no_accesses(self):
128
+ # lifetime should equal idle if never accessed
129
+ c = KVCacheMetricsCollector(sample_rate=1.0)
130
+
131
+ block = KVCacheBlock(block_id=0)
132
+ with patch("time.monotonic_ns", return_value=1000000000):
133
+ c.on_block_allocated(block)
134
+
135
+ with patch("time.monotonic_ns", return_value=6000000000):
136
+ c.on_block_evicted(block)
137
+
138
+ events = c.drain_events()
139
+ assert len(events) == 1
140
+ assert abs(events[0].lifetime_seconds - 5.0) < 0.001
141
+ assert abs(events[0].idle_seconds - 5.0) < 0.001
142
+
143
+ def test_evict(self):
144
+ c = KVCacheMetricsCollector(sample_rate=1.0)
145
+
146
+ block = KVCacheBlock(block_id=0)
147
+ with patch("time.monotonic_ns", return_value=1000000000):
148
+ c.on_block_allocated(block)
149
+
150
+ with patch("time.monotonic_ns", return_value=2000000000):
151
+ c.on_block_accessed(block)
152
+ with patch("time.monotonic_ns", return_value=3000000000):
153
+ c.on_block_accessed(block)
154
+
155
+ with patch("time.monotonic_ns", return_value=4000000000):
156
+ c.on_block_evicted(block)
157
+
158
+ events = c.drain_events()
159
+ assert len(events) == 1
160
+ sample = events[0]
161
+ assert abs(sample.lifetime_seconds - 3.0) < 0.001
162
+ assert abs(sample.idle_seconds - 1.0) < 0.001
163
+ assert sample.reuse_gaps_seconds == (1.0,)
164
+ assert 0 not in c.block_metrics
165
+
166
+ def test_reset(self):
167
+ c = KVCacheMetricsCollector(sample_rate=1.0)
168
+
169
+ with patch("time.monotonic_ns", return_value=1000000000):
170
+ for i in range(5):
171
+ c.on_block_allocated(KVCacheBlock(block_id=i))
172
+
173
+ assert len(c.block_metrics) == 5
174
+ c.reset()
175
+ assert len(c.block_metrics) == 0
176
+
177
+ with patch("time.monotonic_ns", return_value=2000000000):
178
+ c.on_block_allocated(KVCacheBlock(block_id=10))
179
+ assert 10 in c.block_metrics
180
+
181
+ def test_huge_time_jump(self):
182
+ c = KVCacheMetricsCollector(sample_rate=1.0)
183
+
184
+ block = KVCacheBlock(block_id=0)
185
+ with patch("time.monotonic_ns", return_value=1000000000):
186
+ c.on_block_allocated(block)
187
+
188
+ with patch("time.monotonic_ns", return_value=9999999999999999):
189
+ c.on_block_evicted(block)
190
+
191
+ events = c.drain_events()
192
+ assert len(events) == 1
193
+ assert events[0].lifetime_seconds > 0
194
+
195
+
196
+def test_kv_cache_metrics_collector_smoke() -> None:
197
+ """Simple smoke test for KVCacheMetricsCollector on CPU."""
198
+ collector = KVCacheMetricsCollector(sample_rate=1.0)
199
+ block = KVCacheBlock(block_id=123)
200
+
201
+ # Allocate at t = 1.0s.
202
+ with patch("time.monotonic_ns", return_value=1_000_000_000):
203
+ collector.on_block_allocated(block)
204
+
205
+ # Access at t = 2.0s and t = 3.0s.
206
+ with patch("time.monotonic_ns", return_value=2_000_000_000):
207
+ collector.on_block_accessed(block)
208
+ with patch("time.monotonic_ns", return_value=3_000_000_000):
209
+ collector.on_block_accessed(block)
210
+
211
+ # Evict at t = 4.0s.
212
+ with patch("time.monotonic_ns", return_value=4_000_000_000):
213
+ collector.on_block_evicted(block)
214
+
215
+ events = collector.drain_events()
216
+ assert len(events) == 1
217
+
218
+ event = events[0]
219
+ # Lifetime: 1.0s → 4.0s.
220
+ assert abs(event.lifetime_seconds - 3.0) < 1e-6
221
+ # Idle: last access at 3.0s, evicted at 4.0s.
222
+ assert abs(event.idle_seconds - 1.0) < 1e-6
223
+ # One reuse gap between the two accesses.
224
+ assert event.reuse_gaps_seconds == (1.0,)
vllm/config/observability.py
+9
-1
@@ -5,7 +5,7 @@ from functools import cached_property
5
from typing import Any, Literal, cast
6
7
from packaging.version import parse
8
-from pydantic import field_validator, model_validator
8
+from pydantic import Field, field_validator, model_validator
9
from pydantic.dataclasses import dataclass
10
11
from vllm import version
@@ -47,6 +47,14 @@ class ObservabilityConfig:
47
Note that collecting detailed timing information for each request can be
48
expensive."""
49
50
+ kv_cache_metrics: bool = False
51
+ """Enable KV cache residency metrics (lifetime, idle time, reuse gaps).
52
+ Uses sampling to minimize overhead.
53
+ Requires log stats to be enabled (i.e., --disable-log-stats not set)."""
54
+
55
+ kv_cache_metrics_sample: float = Field(default=0.01, gt=0, le=1)
56
+ """Sampling rate for KV cache metrics (0.0, 1.0]. Default 0.01 = 1% of blocks."""
57
+
58
@cached_property
59
def collect_model_forward_time(self) -> bool:
60
"""Whether to collect model forward time for the request."""
vllm/engine/arg_utils.py
+13
@@ -517,6 +517,10 @@ class EngineArgs:
517
collect_detailed_traces: list[DetailedTraceModules] | None = (
518
ObservabilityConfig.collect_detailed_traces
519
)
520
+ kv_cache_metrics: bool = ObservabilityConfig.kv_cache_metrics
521
+ kv_cache_metrics_sample: float = get_field(
522
+ ObservabilityConfig, "kv_cache_metrics_sample"
523
+ )
524
scheduling_policy: SchedulerPolicy = SchedulerConfig.policy
525
scheduler_cls: str | type[object] | None = SchedulerConfig.scheduler_cls
526
@@ -1013,6 +1017,13 @@ class EngineArgs:
1017
"--collect-detailed-traces",
1018
**observability_kwargs["collect_detailed_traces"],
1019
)
1020
+ observability_group.add_argument(
1021
+ "--kv-cache-metrics", **observability_kwargs["kv_cache_metrics"]
1022
+ )
1023
+ observability_group.add_argument(
1024
+ "--kv-cache-metrics-sample",
1025
+ **observability_kwargs["kv_cache_metrics_sample"],
1026
+ )
1027
1028
# Scheduler arguments
1029
scheduler_kwargs = get_kwargs(SchedulerConfig)
@@ -1698,6 +1709,8 @@ class EngineArgs:
1709
show_hidden_metrics_for_version=self.show_hidden_metrics_for_version,
1710
otlp_traces_endpoint=self.otlp_traces_endpoint,
1711
collect_detailed_traces=self.collect_detailed_traces,
1712
+ kv_cache_metrics=self.kv_cache_metrics,
1713
+ kv_cache_metrics_sample=self.kv_cache_metrics_sample,
1714
)
1715
1716
# Compilation config overrides
vllm/v1/core/block_pool.py
+18
@@ -11,6 +11,7 @@ from vllm.distributed.kv_events import (
11
KVCacheEvent,
12
)
13
from vllm.logger import init_logger
14
+from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
15
from vllm.v1.core.kv_cache_utils import (
16
BlockHash,
17
BlockHashList,
@@ -140,6 +141,7 @@ class BlockPool:
141
where different KV cache groups have different block sizes, the
142
actual block size can be a multiple of hash_block_size.
143
enable_kv_cache_events: Whether to enable kv cache events.
144
+ metrics_collector: Optional metrics collector for tracking block residency.
145
"""
146
147
def __init__(
@@ -148,6 +150,7 @@ class BlockPool:
150
enable_caching: bool,
151
hash_block_size: int,
152
enable_kv_cache_events: bool = False,
153
+ metrics_collector: KVCacheMetricsCollector | None = None,
154
):
155
assert isinstance(num_gpu_blocks, int) and num_gpu_blocks > 0
156
self.num_gpu_blocks = num_gpu_blocks
@@ -174,6 +177,8 @@ class BlockPool:
177
self.enable_kv_cache_events = enable_kv_cache_events
178
self.kv_event_queue: list[KVCacheEvent] = []
179
180
+ self.metrics_collector = metrics_collector
181
+
182
def get_cached_block(
183
self, block_hash: BlockHash, kv_cache_group_ids: list[int]
184
) -> list[KVCacheBlock] | None:
@@ -308,10 +313,14 @@ class BlockPool:
313
self._maybe_evict_cached_block(block)
314
assert block.ref_cnt == 0
315
block.ref_cnt += 1
316
+ if self.metrics_collector:
317
+ self.metrics_collector.on_block_allocated(block)
318
else:
319
for block in ret:
320
assert block.ref_cnt == 0
321
block.ref_cnt += 1
322
+ if self.metrics_collector:
323
+ self.metrics_collector.on_block_allocated(block)
324
return ret
325
326
def _maybe_evict_cached_block(self, block: KVCacheBlock) -> bool:
@@ -325,6 +334,10 @@ class BlockPool:
334
Returns:
335
True if the block is evicted, False otherwise.
336
"""
337
+ # Clean up metrics tracking first to prevent leaks
338
+ if self.metrics_collector:
339
+ self.metrics_collector.on_block_evicted(block)
340
+
341
block_hash = block.block_hash
342
if block_hash is None:
343
# The block doesn't have hash, eviction is not needed
@@ -365,6 +378,8 @@ class BlockPool:
378
if block.ref_cnt == 0 and not block.is_null:
379
self.free_block_queue.remove(block)
380
block.ref_cnt += 1
381
+ if self.metrics_collector:
382
+ self.metrics_collector.on_block_accessed(block)
383
384
def free_blocks(self, ordered_blocks: Iterable[KVCacheBlock]) -> None:
385
"""Free a list of blocks. The blocks should be ordered by their
@@ -407,6 +422,9 @@ class BlockPool:
422
for block in self.blocks:
423
block.reset_hash()
424
425
+ if self.metrics_collector:
426
+ self.metrics_collector.reset()
427
+
428
logger.info("Successfully reset prefix cache")
429
430
if self.enable_kv_cache_events:
vllm/v1/core/kv_cache_coordinator.py
+22
-9
@@ -5,6 +5,7 @@ from collections.abc import Sequence
5
from math import lcm
6
7
from vllm.v1.core.block_pool import BlockPool
8
+from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
9
from vllm.v1.core.kv_cache_utils import (
10
BlockHash,
11
BlockHashList,
@@ -39,6 +40,7 @@ class KVCacheCoordinator(ABC):
40
dcp_world_size: int,
41
pcp_world_size: int,
42
hash_block_size: int,
43
+ metrics_collector: KVCacheMetricsCollector | None = None,
44
):
45
self.kv_cache_config = kv_cache_config
46
self.max_model_len = max_model_len
@@ -49,6 +51,7 @@ class KVCacheCoordinator(ABC):
51
enable_caching,
52
hash_block_size,
53
enable_kv_cache_events,
54
+ metrics_collector,
55
)
56
57
# Needs special handling for find_longest_cache_hit if eagle is enabled
@@ -228,6 +231,7 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
231
dcp_world_size: int,
232
pcp_world_size: int,
233
hash_block_size: int,
234
+ metrics_collector: KVCacheMetricsCollector | None = None,
235
):
236
super().__init__(
237
kv_cache_config,
@@ -238,6 +242,7 @@ class KVCacheCoordinatorNoPrefixCache(KVCacheCoordinator):
242
dcp_world_size=dcp_world_size,
243
pcp_world_size=pcp_world_size,
244
hash_block_size=hash_block_size,
245
+ metrics_collector=metrics_collector,
246
)
247
self.num_single_type_manager = len(self.single_type_managers)
248
@@ -272,6 +277,7 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
277
dcp_world_size: int,
278
pcp_world_size: int,
279
hash_block_size: int,
280
+ metrics_collector: KVCacheMetricsCollector | None = None,
281
):
282
super().__init__(
283
kv_cache_config,
@@ -282,6 +288,7 @@ class UnitaryKVCacheCoordinator(KVCacheCoordinator):
288
dcp_world_size=dcp_world_size,
289
pcp_world_size=pcp_world_size,
290
hash_block_size=hash_block_size,
291
+ metrics_collector=metrics_collector,
292
)
293
self.kv_cache_spec = self.kv_cache_config.kv_cache_groups[0].kv_cache_spec
294
self.block_size = self.kv_cache_spec.block_size
@@ -338,6 +345,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
345
dcp_world_size: int,
346
pcp_world_size: int,
347
hash_block_size: int,
348
+ metrics_collector: KVCacheMetricsCollector | None = None,
349
):
350
super().__init__(
351
kv_cache_config,
@@ -348,6 +356,7 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
356
dcp_world_size=dcp_world_size,
357
pcp_world_size=pcp_world_size,
358
hash_block_size=hash_block_size,
359
+ metrics_collector=metrics_collector,
360
)
361
# hash_block_size: the block size used to compute block hashes.
362
# The actual block size usually equals hash_block_size, but in cases where
@@ -523,6 +532,7 @@ def get_kv_cache_coordinator(
532
dcp_world_size: int,
533
pcp_world_size: int,
534
hash_block_size: int,
535
+ metrics_collector: KVCacheMetricsCollector | None = None,
536
) -> KVCacheCoordinator:
537
if not enable_caching:
538
return KVCacheCoordinatorNoPrefixCache(
@@ -530,9 +540,10 @@ def get_kv_cache_coordinator(
540
max_model_len,
541
use_eagle,
542
enable_kv_cache_events,
533
- dcp_world_size,
534
- pcp_world_size,
535
- hash_block_size,
543
+ dcp_world_size=dcp_world_size,
544
+ pcp_world_size=pcp_world_size,
545
+ hash_block_size=hash_block_size,
546
+ metrics_collector=metrics_collector,
547
)
548
if len(kv_cache_config.kv_cache_groups) == 1:
549
return UnitaryKVCacheCoordinator(
@@ -541,9 +552,10 @@ def get_kv_cache_coordinator(
552
use_eagle,
553
enable_caching,
554
enable_kv_cache_events,
544
- dcp_world_size,
545
- pcp_world_size,
546
- hash_block_size,
555
+ dcp_world_size=dcp_world_size,
556
+ pcp_world_size=pcp_world_size,
557
+ hash_block_size=hash_block_size,
558
+ metrics_collector=metrics_collector,
559
)
560
return HybridKVCacheCoordinator(
561
kv_cache_config,
@@ -551,7 +563,8 @@ def get_kv_cache_coordinator(
563
use_eagle,
564
enable_caching,
565
enable_kv_cache_events,
554
- dcp_world_size,
555
- pcp_world_size,
556
- hash_block_size,
566
+ dcp_world_size=dcp_world_size,
567
+ pcp_world_size=pcp_world_size,
568
+ hash_block_size=hash_block_size,
569
+ metrics_collector=metrics_collector,
570
)
vllm/v1/core/kv_cache_manager.py
+4
@@ -9,6 +9,7 @@ from typing import Literal, overload
9
from vllm.distributed.kv_events import KVCacheEvent
10
from vllm.logger import init_logger
11
from vllm.v1.core.kv_cache_coordinator import get_kv_cache_coordinator
12
+from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
13
from vllm.v1.core.kv_cache_utils import KVCacheBlock
14
from vllm.v1.kv_cache_interface import KVCacheConfig
15
from vllm.v1.metrics.stats import PrefixCacheStats
@@ -102,12 +103,14 @@ class KVCacheManager:
103
enable_kv_cache_events: bool = False,
104
dcp_world_size: int = 1,
105
pcp_world_size: int = 1,
106
+ metrics_collector: KVCacheMetricsCollector | None = None,
107
) -> None:
108
self.max_model_len = max_model_len
109
110
self.enable_caching = enable_caching
111
self.use_eagle = use_eagle
112
self.log_stats = log_stats
113
+ self.metrics_collector = metrics_collector
114
# FIXME: make prefix cache stats conditional on log_stats. We still need
115
# this comment because when the log stats is enabled there are still
116
# potential configs we could expose in the future.
@@ -122,6 +125,7 @@ class KVCacheManager:
125
dcp_world_size=dcp_world_size,
126
pcp_world_size=pcp_world_size,
127
hash_block_size=hash_block_size,
128
+ metrics_collector=self.metrics_collector,
129
)
130
self.num_kv_cache_groups = len(kv_cache_config.kv_cache_groups)
131
self.block_pool = self.coordinator.block_pool
vllm/v1/core/kv_cache_metrics.py
new
+96
@@ -0,0 +1,96 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""KV cache metrics tracking."""
4
+
5
+import random
6
+import time
7
+from collections import deque
8
+from typing import TYPE_CHECKING
9
+
10
+if TYPE_CHECKING:
11
+ from vllm.v1.core.kv_cache_utils import KVCacheBlock
12
+
13
+from vllm.v1.metrics.stats import KVCacheEvictionEvent
14
+
15
+
16
+class BlockMetricsState:
17
+ """Tracks lifecycle metrics for a single KV cache block."""
18
+
19
+ def __init__(self):
20
+ now_ns = time.monotonic_ns()
21
+ self.birth_time_ns = now_ns
22
+ self.last_access_ns = now_ns
23
+ # Bounded to prevent unbounded growth if a block is accessed many times.
24
+ self.access_history: deque[int] = deque(maxlen=4)
25
+
26
+ def record_access(self) -> None:
27
+ now_ns = time.monotonic_ns()
28
+ self.last_access_ns = now_ns
29
+ self.access_history.append(now_ns)
30
+
31
+ def get_lifetime_seconds(self) -> float:
32
+ now_ns = time.monotonic_ns()
33
+ return (now_ns - self.birth_time_ns) / 1e9
34
+
35
+ def get_idle_time_seconds(self) -> float:
36
+ now_ns = time.monotonic_ns()
37
+ return (now_ns - self.last_access_ns) / 1e9
38
+
39
+ def get_reuse_gaps_seconds(self) -> list[float]:
40
+ if len(self.access_history) < 2:
41
+ return []
42
+ history = list(self.access_history)
43
+ return [(history[i] - history[i - 1]) / 1e9 for i in range(1, len(history))]
44
+
45
+
46
+class KVCacheMetricsCollector:
47
+ """Collects KV cache residency metrics with sampling."""
48
+
49
+ def __init__(self, sample_rate: float = 0.01):
50
+ assert 0 < sample_rate <= 1.0, (
51
+ f"sample_rate must be in (0, 1.0], got {sample_rate}"
52
+ )
53
+ self.sample_rate = sample_rate
54
+
55
+ self.block_metrics: dict[int, BlockMetricsState] = {}
56
+
57
+ self._eviction_events: list[KVCacheEvictionEvent] = []
58
+
59
+ def should_sample_block(self) -> bool:
60
+ return random.random() < self.sample_rate
61
+
62
+ def on_block_allocated(self, block: "KVCacheBlock") -> None:
63
+ if self.should_sample_block():
64
+ self.block_metrics[block.block_id] = BlockMetricsState()
65
+
66
+ def on_block_accessed(self, block: "KVCacheBlock") -> None:
67
+ metrics = self.block_metrics.get(block.block_id)
68
+ if metrics:
69
+ metrics.record_access()
70
+
71
+ def on_block_evicted(self, block: "KVCacheBlock") -> None:
72
+ metrics = self.block_metrics.pop(block.block_id, None)
73
+ if not metrics:
74
+ return
75
+
76
+ lifetime = metrics.get_lifetime_seconds()
77
+ idle_time = metrics.get_idle_time_seconds()
78
+ reuse_gaps = tuple(metrics.get_reuse_gaps_seconds())
79
+
80
+ self._eviction_events.append(
81
+ KVCacheEvictionEvent(
82
+ lifetime_seconds=lifetime,
83
+ idle_seconds=idle_time,
84
+ reuse_gaps_seconds=reuse_gaps,
85
+ )
86
+ )
87
+
88
+ def reset(self) -> None:
89
+ """Clear all state on cache reset."""
90
+ self.block_metrics.clear()
91
+ self._eviction_events.clear()
92
+
93
+ def drain_events(self) -> list[KVCacheEvictionEvent]:
94
+ events = self._eviction_events
95
+ self._eviction_events = []
96
+ return events
vllm/v1/core/sched/scheduler.py
+24
-3
@@ -29,6 +29,7 @@ from vllm.v1.core.encoder_cache_manager import (
29
compute_encoder_budget,
30
)
31
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
32
+from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector
33
from vllm.v1.core.sched.interface import SchedulerInterface
34
from vllm.v1.core.sched.output import (
35
CachedRequestData,
@@ -40,7 +41,10 @@ from vllm.v1.core.sched.request_queue import SchedulingPolicy, create_request_qu
41
from vllm.v1.core.sched.utils import check_stop, remove_all
42
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
43
from vllm.v1.kv_cache_interface import KVCacheConfig
43
-from vllm.v1.metrics.stats import PrefixCacheStats, SchedulerStats
44
+from vllm.v1.metrics.stats import (
45
+ PrefixCacheStats,
46
+ SchedulerStats,
47
+)
48
from vllm.v1.outputs import DraftTokenIds, KVConnectorOutput, ModelRunnerOutput
49
from vllm.v1.request import Request, RequestStatus
50
from vllm.v1.spec_decode.metrics import SpecDecodingStats
@@ -69,6 +73,12 @@ class Scheduler(SchedulerInterface):
73
self.kv_events_config = vllm_config.kv_events_config
74
self.parallel_config = vllm_config.parallel_config
75
self.log_stats = log_stats
76
+ self.observability_config = vllm_config.observability_config
77
+ self.kv_metrics_collector: KVCacheMetricsCollector | None = None
78
+ if self.observability_config.kv_cache_metrics:
79
+ self.kv_metrics_collector = KVCacheMetricsCollector(
80
+ self.observability_config.kv_cache_metrics_sample,
81
+ )
82
self.structured_output_manager = structured_output_manager
83
self.is_encoder_decoder = vllm_config.model_config.is_encoder_decoder
84
@@ -187,6 +197,7 @@ class Scheduler(SchedulerInterface):
197
dcp_world_size=self.dcp_world_size,
198
pcp_world_size=self.pcp_world_size,
199
hash_block_size=self.block_size,
200
+ metrics_collector=self.kv_metrics_collector,
201
)
202
self.use_pp = self.parallel_config.pipeline_parallel_size > 1
203
self.use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER
@@ -1356,14 +1367,24 @@ class Scheduler(SchedulerInterface):
1367
prefix_cache_stats = self.kv_cache_manager.make_prefix_cache_stats()
1368
assert prefix_cache_stats is not None
1369
connector_prefix_cache_stats = self._make_connector_prefix_cache_stats()
1370
+ eviction_events = (
1371
+ self.kv_metrics_collector.drain_events()
1372
+ if self.kv_metrics_collector is not None
1373
+ else []
1374
+ )
1375
+ spec_stats = spec_decoding_stats
1376
+ connector_stats_payload = (
1377
+ kv_connector_stats.data if kv_connector_stats else None
1378
+ )
1379
return SchedulerStats(
1380
num_running_reqs=len(self.running),
1381
num_waiting_reqs=len(self.waiting),
1382
kv_cache_usage=self.kv_cache_manager.usage,
1383
prefix_cache_stats=prefix_cache_stats,
1384
connector_prefix_cache_stats=connector_prefix_cache_stats,
1365
- spec_decoding_stats=spec_decoding_stats,
1366
- kv_connector_stats=kv_connector_stats.data if kv_connector_stats else None,
1385
+ kv_cache_eviction_events=eviction_events,
1386
+ spec_decoding_stats=spec_stats,
1387
+ kv_connector_stats=connector_stats_payload,
1388
)
1389
1390
def make_spec_decoding_stats(
vllm/v1/metrics/loggers.py
+90
@@ -375,6 +375,9 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
375
# Use this flag to hide metrics that were deprecated in
376
# a previous release and which will be removed future
377
self.show_hidden_metrics = vllm_config.observability_config.show_hidden_metrics
378
+ self.kv_cache_metrics_enabled = (
379
+ vllm_config.observability_config.kv_cache_metrics
380
+ )
381
382
labelnames = ["model_name", "engine"]
383
model_name = vllm_config.model_config.served_model_name
@@ -853,6 +856,79 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
856
histogram_decode_time_request, engine_indexes, model_name
857
)
858
859
+ #
860
+ # KV Cache residency metrics
861
+ #
862
+ if self.kv_cache_metrics_enabled:
863
+ kv_cache_residency_buckets = [
864
+ 0.001,
865
+ 0.002,
866
+ 0.005,
867
+ 0.01,
868
+ 0.02,
869
+ 0.05,
870
+ 0.1,
871
+ 0.2,
872
+ 0.5,
873
+ 1,
874
+ 2,
875
+ 5,
876
+ 10,
877
+ 20,
878
+ 30,
879
+ 60,
880
+ 120,
881
+ 300,
882
+ 600,
883
+ 1200,
884
+ 1800,
885
+ ]
886
+
887
+ histogram_kv_block_lifetime = self._histogram_cls(
888
+ name="vllm:kv_block_lifetime_seconds",
889
+ documentation=(
890
+ "Histogram of KV cache block lifetime from allocation to eviction. "
891
+ "Sampled metrics (controlled by --kv-cache-metrics-sample)."
892
+ ),
893
+ buckets=kv_cache_residency_buckets,
894
+ labelnames=labelnames,
895
+ )
896
+ self.histogram_kv_block_lifetime = make_per_engine(
897
+ histogram_kv_block_lifetime, engine_indexes, model_name
898
+ )
899
+
900
+ histogram_kv_block_idle_before_evict = self._histogram_cls(
901
+ name="vllm:kv_block_idle_before_evict_seconds",
902
+ documentation=(
903
+ "Histogram of idle time before KV cache block eviction. "
904
+ "Sampled metrics (controlled by --kv-cache-metrics-sample)."
905
+ ),
906
+ buckets=kv_cache_residency_buckets,
907
+ labelnames=labelnames,
908
+ )
909
+ self.histogram_kv_block_idle_before_evict = make_per_engine(
910
+ histogram_kv_block_idle_before_evict, engine_indexes, model_name
911
+ )
912
+
913
+ histogram_kv_block_reuse_gap = self._histogram_cls(
914
+ name="vllm:kv_block_reuse_gap_seconds",
915
+ documentation=(
916
+ "Histogram of time gaps between consecutive KV cache block "
917
+ "accesses. Only the most recent accesses are recorded "
918
+ "(ring buffer). Sampled metrics (controlled by "
919
+ "--kv-cache-metrics-sample)."
920
+ ),
921
+ buckets=kv_cache_residency_buckets,
922
+ labelnames=labelnames,
923
+ )
924
+ self.histogram_kv_block_reuse_gap = make_per_engine(
925
+ histogram_kv_block_reuse_gap, engine_indexes, model_name
926
+ )
927
+ else:
928
+ self.histogram_kv_block_lifetime = {}
929
+ self.histogram_kv_block_idle_before_evict = {}
930
+ self.histogram_kv_block_reuse_gap = {}
931
+
932
#
933
# LoRA metrics
934
#
@@ -944,6 +1020,20 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
1020
scheduler_stats.kv_connector_stats, engine_idx
1021
)
1022
1023
+ if (
1024
+ self.kv_cache_metrics_enabled
1025
+ and scheduler_stats.kv_cache_eviction_events
1026
+ ):
1027
+ lifetime_hist = self.histogram_kv_block_lifetime[engine_idx]
1028
+ idle_hist = self.histogram_kv_block_idle_before_evict[engine_idx]
1029
+ reuse_hist = self.histogram_kv_block_reuse_gap[engine_idx]
1030
+
1031
+ for event in scheduler_stats.kv_cache_eviction_events:
1032
+ lifetime_hist.observe(event.lifetime_seconds)
1033
+ idle_hist.observe(event.idle_seconds)
1034
+ for gap in event.reuse_gaps_seconds:
1035
+ reuse_hist.observe(gap)
1036
+
1037
if self.gauge_lora_info is not None:
1038
running_lora_adapters = ",".join(
1039
scheduler_stats.running_lora_adapters.keys()
vllm/v1/metrics/stats.py
+11
@@ -150,6 +150,15 @@ class MultiModalCacheStats(BaseCacheStats):
150
"""
151
152
153
+@dataclass
154
+class KVCacheEvictionEvent:
155
+ """Single KV cache block eviction sample."""
156
+
157
+ lifetime_seconds: float
158
+ idle_seconds: float
159
+ reuse_gaps_seconds: tuple[float, ...]
160
+
161
+
162
@dataclass
163
class SchedulerStats:
164
"""Stats associated with the scheduler."""
@@ -166,6 +175,8 @@ class SchedulerStats:
175
prefix_cache_stats: PrefixCacheStats = field(default_factory=PrefixCacheStats)
176
connector_prefix_cache_stats: PrefixCacheStats | None = None
177
178
+ kv_cache_eviction_events: list[KVCacheEvictionEvent] = field(default_factory=list)
179
+
180
spec_decoding_stats: SpecDecodingStats | None = None
181
kv_connector_stats: dict[str, Any] | None = None
182