A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1][Metrics] Add GPU prefix cache hit rate % gauge (#12592)
Cody Yu committed
Feb 11, 2025 at 00:27 UTC
41c5dd45b98d5a6facad328a1ce534b9a94763a2
7 files changed
+174
-5
tests/entrypoints/openai/test_metrics.py
+2
@@ -203,6 +203,8 @@ EXPECTED_METRICS_V1 = [
203
"vllm:num_requests_running",
204
"vllm:num_requests_waiting",
205
"vllm:gpu_cache_usage_perc",
206
+ "vllm:gpu_prefix_cache_queries",
207
+ "vllm:gpu_prefix_cache_hits",
208
"vllm:prompt_tokens_total",
209
"vllm:generation_tokens_total",
210
"vllm:request_success_total",
tests/v1/core/test_kv_cache_utils.py
+38
-1
@@ -5,10 +5,11 @@ import pytest
5
from vllm.multimodal.inputs import MultiModalKwargs
6
from vllm.sampling_params import SamplingParams
7
from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,
8
- KVCacheBlock,
8
+ KVCacheBlock, PrefixCachingMetrics,
9
generate_block_hash_extra_keys,
10
hash_block_tokens,
11
hash_request_tokens)
12
+from vllm.v1.metrics.stats import PrefixCacheStats
13
from vllm.v1.request import Request
14
15
@@ -277,3 +278,39 @@ def test_hash_request_tokens_no_mm_inputs():
278
assert block_hashes[0].extra_keys is None
279
assert block_hashes[1].token_ids == (3, 4, 5)
280
assert block_hashes[1].extra_keys is None
281
+
282
+
283
+def test_metrics():
284
+ """
285
+ Test the prefix caching metrics.
286
+ """
287
+
288
+ def stats(requests, queries, hits):
289
+ return PrefixCacheStats(requests=requests, queries=queries, hits=hits)
290
+
291
+ metrics = PrefixCachingMetrics(interval=5)
292
+ assert metrics.hit_rate == 0.0
293
+
294
+ metrics.observe(stats(1, 20, 9))
295
+ # 9 / 20 = 0.45
296
+ assert metrics.hit_rate == 0.45
297
+
298
+ metrics.observe(stats(4, 80, 16))
299
+
300
+ # 25 / 100 = 0.25
301
+ assert metrics.hit_rate == 0.25
302
+
303
+ metrics.observe(stats(1, 10, 2))
304
+
305
+ # Remove (20, 9) and add (10, 2): 18 / 90 = 0.2
306
+ assert metrics.aggregated_requests == 5
307
+ assert metrics.aggregated_query_total == 90
308
+ assert metrics.aggregated_query_hit == 18
309
+ assert metrics.hit_rate == 0.2
310
+
311
+ metrics.reset()
312
+ assert metrics.hit_rate == 0.0
313
+ assert metrics.aggregated_requests == 0
314
+ assert metrics.aggregated_query_total == 0
315
+ assert metrics.aggregated_query_hit == 0
316
+ assert not metrics.query_queue
vllm/v1/core/kv_cache_manager.py
+24
@@ -10,6 +10,7 @@ from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,
10
generate_block_hash_extra_keys,
11
hash_block_tokens,
12
hash_request_tokens)
13
+from vllm.v1.metrics.stats import PrefixCacheStats
14
from vllm.v1.request import Request, RequestStatus
15
16
logger = init_logger(__name__)
@@ -78,11 +79,28 @@ class KVCacheManager:
79
self.req_to_block_hashes: DefaultDict[
80
str, List[BlockHashType]] = defaultdict(list)
81
82
+ self.prefix_cache_stats = PrefixCacheStats()
83
+
84
@property
85
def usage(self) -> float:
86
+ """Get the KV cache usage.
87
+
88
+ Returns:
89
+ The KV cache usage (between 0.0 and 1.0).
90
+ """
91
return 1.0 - (self.free_block_queue.num_free_blocks /
92
self.num_gpu_blocks)
93
94
+ def make_prefix_cache_stats(self) -> PrefixCacheStats:
95
+ """Get (and reset) the prefix cache stats.
96
+
97
+ Returns:
98
+ The current prefix caching stats.
99
+ """
100
+ stats = self.prefix_cache_stats
101
+ self.prefix_cache_stats = PrefixCacheStats()
102
+ return stats
103
+
104
def get_computed_blocks(
105
self, request: Request) -> Tuple[List[KVCacheBlock], int]:
106
"""Get the computed (cached) blocks for the request.
@@ -118,6 +136,10 @@ class KVCacheManager:
136
else:
137
break
138
139
+ self.prefix_cache_stats.requests += 1
140
+ self.prefix_cache_stats.queries += len(block_hashes)
141
+ self.prefix_cache_stats.hits += len(computed_blocks)
142
+
143
# NOTE(woosuk): Since incomplete blocks are not eligible for
144
# sharing, `num_computed_tokens` is always a multiple of
145
# `block_size`.
@@ -280,6 +302,8 @@ class KVCacheManager:
302
for block in self.block_pool:
303
block.reset_hash()
304
305
+ self.prefix_cache_stats.reset = True
306
+
307
logger.info("Successfully reset prefix cache")
308
return True
309
vllm/v1/core/kv_cache_utils.py
+64
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""KV-Cache Utilities."""
3
+from collections import deque
4
from collections.abc import Sequence
5
from dataclasses import dataclass
6
from typing import Any, List, NamedTuple, Optional, Tuple
@@ -8,6 +9,7 @@ from vllm.config import VllmConfig
9
from vllm.logger import init_logger
10
from vllm.v1.kv_cache_interface import (KVCacheConfig, KVCacheSpec,
11
KVCacheTensor)
12
+from vllm.v1.metrics.stats import PrefixCacheStats
13
from vllm.v1.request import Request
14
15
logger = init_logger(__name__)
@@ -28,6 +30,68 @@ class BlockHashType(NamedTuple):
30
extra_keys: Optional[Any] = None
31
32
33
+class PrefixCachingMetrics:
34
+ """Metrics for prefix caching with a hit rate of the most recent N requests.
35
+
36
+ Args:
37
+ interval: The number of the most recent requests to aggregate.
38
+ Defaults to 1000.
39
+ """
40
+
41
+ def __init__(self, interval: int = 1000):
42
+ self.interval = interval
43
+ # The current aggregated values.
44
+ self.aggregated_requests = 0
45
+ self.aggregated_query_total = 0
46
+ self.aggregated_query_hit = 0
47
+ # A deque of (requests, queries, hits) for the most recent requests.
48
+ self.query_queue: deque[Tuple[int, int, int]] = deque()
49
+
50
+ def observe(self, stats: PrefixCacheStats):
51
+ """Observe the prefix caching for a set of requests.
52
+
53
+ This function is called with information gathered when new requests
54
+ are being scheduled and are looking for computed blocks.
55
+
56
+ When there are more than `interval` requests, the oldest set of
57
+ requestsare removed from the metrics.
58
+
59
+ Args:
60
+ stats: The prefix cache stats.
61
+ """
62
+ # reset_prefix_cache was invoked before the current update.
63
+ # Reset the metrics before aggregating the current stats.
64
+ if stats.reset:
65
+ self.reset()
66
+
67
+ # Update the metrics.
68
+ self.query_queue.append((stats.requests, stats.queries, stats.hits))
69
+ self.aggregated_requests += stats.requests
70
+ self.aggregated_query_total += stats.queries
71
+ self.aggregated_query_hit += stats.hits
72
+
73
+ # Remove the oldest stats if the number of requests exceeds.
74
+ if self.aggregated_requests > self.interval:
75
+ old_requests, old_queries, old_hits = self.query_queue.popleft()
76
+ self.aggregated_requests -= old_requests
77
+ self.aggregated_query_total -= old_queries
78
+ self.aggregated_query_hit -= old_hits
79
+
80
+ def reset(self):
81
+ """Reset the metrics."""
82
+ self.aggregated_requests = 0
83
+ self.aggregated_query_total = 0
84
+ self.aggregated_query_hit = 0
85
+ self.query_queue.clear()
86
+
87
+ @property
88
+ def hit_rate(self) -> float:
89
+ """Calculate the hit rate for the past N requests."""
90
+ if self.aggregated_query_total == 0:
91
+ return 0.0
92
+ return self.aggregated_query_hit / self.aggregated_query_total
93
+
94
+
95
@dataclass
96
class KVCacheBlock:
97
"""KV-cache block metadata."""
vllm/v1/core/scheduler.py
+1
@@ -593,4 +593,5 @@ class Scheduler:
593
num_running_reqs=len(self.running),
594
num_waiting_reqs=len(self.waiting),
595
gpu_cache_usage=self.kv_cache_manager.usage,
596
+ prefix_cache_stats=self.kv_cache_manager.make_prefix_cache_stats(),
597
)
vllm/v1/metrics/loggers.py
+27
-2
@@ -9,6 +9,7 @@ import prometheus_client
9
10
from vllm.config import ModelConfig
11
from vllm.logger import init_logger
12
+from vllm.v1.core.kv_cache_utils import PrefixCachingMetrics
13
from vllm.v1.engine import FinishReason
14
from vllm.v1.metrics.stats import IterationStats, SchedulerStats
15
@@ -37,6 +38,9 @@ class LoggingStatLogger(StatLoggerBase):
38
self.num_prompt_tokens: List[int] = []
39
self.num_generation_tokens: List[int] = []
40
41
+ # Prefix cache metrics. TODO: Make the interval configurable.
42
+ self.prefix_caching_metrics = PrefixCachingMetrics()
43
+
44
def _local_interval_elapsed(self, now: float) -> bool:
45
# Log every _LOCAL_LOGGING_INTERVAL_SEC.
46
elapsed_time = now - self.last_log_time
@@ -58,6 +62,8 @@ class LoggingStatLogger(StatLoggerBase):
62
63
self._track_iteration_stats(iteration_stats)
64
65
+ self.prefix_caching_metrics.observe(scheduler_stats.prefix_cache_stats)
66
+
67
now = time.monotonic()
68
if not self._local_interval_elapsed(now):
69
return
@@ -72,13 +78,15 @@ class LoggingStatLogger(StatLoggerBase):
78
logger.info(
79
"Avg prompt throughput: %.1f tokens/s, "
80
"Avg generation throughput: %.1f tokens/s, "
75
- "Running: %d reqs, Waiting: %d reqs "
76
- "GPU KV cache usage: %.1f%%.",
81
+ "Running: %d reqs, Waiting: %d reqs, "
82
+ "GPU KV cache usage: %.1f%%, "
83
+ "Prefix cache hit rate: %.1f%%",
84
prompt_throughput,
85
generation_throughput,
86
scheduler_stats.num_running_reqs,
87
scheduler_stats.num_waiting_reqs,
88
scheduler_stats.gpu_cache_usage * 100,
89
+ self.prefix_caching_metrics.hit_rate * 100,
90
)
91
92
@@ -107,6 +115,18 @@ class PrometheusStatLogger(StatLoggerBase):
115
documentation="GPU KV-cache usage. 1 means 100 percent usage.",
116
labelnames=labelnames).labels(*labelvalues)
117
118
+ self.counter_gpu_prefix_cache_queries = prometheus_client.Counter(
119
+ name="vllm:gpu_prefix_cache_queries",
120
+ documentation=
121
+ "GPU prefix cache queries, in terms of number of queried blocks.",
122
+ labelnames=labelnames).labels(*labelvalues)
123
+
124
+ self.counter_gpu_prefix_cache_hits = prometheus_client.Counter(
125
+ name="vllm:gpu_prefix_cache_hits",
126
+ documentation=
127
+ "GPU prefix cache hits, in terms of number of cached blocks.",
128
+ labelnames=labelnames).labels(*labelvalues)
129
+
130
self.counter_prompt_tokens = prometheus_client.Counter(
131
name="vllm:prompt_tokens_total",
132
documentation="Number of prefill tokens processed.",
@@ -170,6 +190,11 @@ class PrometheusStatLogger(StatLoggerBase):
190
191
self.gauge_gpu_cache_usage.set(scheduler_stats.gpu_cache_usage)
192
193
+ self.counter_gpu_prefix_cache_queries.inc(
194
+ scheduler_stats.prefix_cache_stats.queries)
195
+ self.counter_gpu_prefix_cache_hits.inc(
196
+ scheduler_stats.prefix_cache_stats.hits)
197
+
198
self.counter_prompt_tokens.inc(iteration_stats.num_prompt_tokens)
199
self.counter_generation_tokens.inc(
200
iteration_stats.num_generation_tokens)
vllm/v1/metrics/stats.py
+18
-2
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import time
4
-from dataclasses import dataclass
4
+from dataclasses import dataclass, field
5
from typing import TYPE_CHECKING, List
6
7
if TYPE_CHECKING:
@@ -9,6 +9,20 @@ if TYPE_CHECKING:
9
from vllm.v1.engine import EngineCoreOutput, FinishReason
10
11
12
+@dataclass
13
+class PrefixCacheStats:
14
+ """Stores prefix cache hit statistics."""
15
+ # Whether reset_prefix_cache was invoked.
16
+ reset: bool = False
17
+ # The number of requests in this update.
18
+ requests: int = 0
19
+ # The number of queries in these requests. Note that "queries" here
20
+ # means the number of blocks that were queried from the cache.
21
+ queries: int = 0
22
+ # The number of hits in these requests.
23
+ hits: int = 0
24
+
25
+
26
@dataclass
27
class SchedulerStats:
28
"""Stats associated with the scheduler."""
@@ -17,7 +31,9 @@ class SchedulerStats:
31
num_waiting_reqs: int = 0
32
33
gpu_cache_usage: float = 0.0
20
- # gpu_prefix_cache_hit_rate: float = 0.0
34
+
35
+ prefix_cache_stats: PrefixCacheStats = field(
36
+ default_factory=PrefixCacheStats)
37
38
39
@dataclass