@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 41c5dd45b9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[V1][Metrics] Add GPU prefix cache hit rate % gauge (#12592)

Cody Yu committed Feb 11, 2025 at 00:27 UTC 41c5dd45b98d5a6facad328a1ce534b9a94763a2
7 files changed +174 -5
tests/entrypoints/openai/test_metrics.py
+2
@@ -203,6 +203,8 @@ EXPECTED_METRICS_V1 = [
203 "vllm:num_requests_running",
204 "vllm:num_requests_waiting",
205 "vllm:gpu_cache_usage_perc",
206 + "vllm:gpu_prefix_cache_queries",
207 + "vllm:gpu_prefix_cache_hits",
208 "vllm:prompt_tokens_total",
209 "vllm:generation_tokens_total",
210 "vllm:request_success_total",
tests/v1/core/test_kv_cache_utils.py
+38 -1
@@ -5,10 +5,11 @@ import pytest
5 from vllm.multimodal.inputs import MultiModalKwargs
6 from vllm.sampling_params import SamplingParams
7 from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,
8 - KVCacheBlock,
8 + KVCacheBlock, PrefixCachingMetrics,
9 generate_block_hash_extra_keys,
10 hash_block_tokens,
11 hash_request_tokens)
12 +from vllm.v1.metrics.stats import PrefixCacheStats
13 from vllm.v1.request import Request
14
15
@@ -277,3 +278,39 @@ def test_hash_request_tokens_no_mm_inputs():
278 assert block_hashes[0].extra_keys is None
279 assert block_hashes[1].token_ids == (3, 4, 5)
280 assert block_hashes[1].extra_keys is None
281 +
282 +
283 +def test_metrics():
284 + """
285 + Test the prefix caching metrics.
286 + """
287 +
288 + def stats(requests, queries, hits):
289 + return PrefixCacheStats(requests=requests, queries=queries, hits=hits)
290 +
291 + metrics = PrefixCachingMetrics(interval=5)
292 + assert metrics.hit_rate == 0.0
293 +
294 + metrics.observe(stats(1, 20, 9))
295 + # 9 / 20 = 0.45
296 + assert metrics.hit_rate == 0.45
297 +
298 + metrics.observe(stats(4, 80, 16))
299 +
300 + # 25 / 100 = 0.25
301 + assert metrics.hit_rate == 0.25
302 +
303 + metrics.observe(stats(1, 10, 2))
304 +
305 + # Remove (20, 9) and add (10, 2): 18 / 90 = 0.2
306 + assert metrics.aggregated_requests == 5
307 + assert metrics.aggregated_query_total == 90
308 + assert metrics.aggregated_query_hit == 18
309 + assert metrics.hit_rate == 0.2
310 +
311 + metrics.reset()
312 + assert metrics.hit_rate == 0.0
313 + assert metrics.aggregated_requests == 0
314 + assert metrics.aggregated_query_total == 0
315 + assert metrics.aggregated_query_hit == 0
316 + assert not metrics.query_queue
vllm/v1/core/kv_cache_manager.py
+24
@@ -10,6 +10,7 @@ from vllm.v1.core.kv_cache_utils import (BlockHashType, FreeKVCacheBlockQueue,
10 generate_block_hash_extra_keys,
11 hash_block_tokens,
12 hash_request_tokens)
13 +from vllm.v1.metrics.stats import PrefixCacheStats
14 from vllm.v1.request import Request, RequestStatus
15
16 logger = init_logger(__name__)
@@ -78,11 +79,28 @@ class KVCacheManager:
79 self.req_to_block_hashes: DefaultDict[
80 str, List[BlockHashType]] = defaultdict(list)
81
82 + self.prefix_cache_stats = PrefixCacheStats()
83 +
84 @property
85 def usage(self) -> float:
86 + """Get the KV cache usage.
87 +
88 + Returns:
89 + The KV cache usage (between 0.0 and 1.0).
90 + """
91 return 1.0 - (self.free_block_queue.num_free_blocks /
92 self.num_gpu_blocks)
93
94 + def make_prefix_cache_stats(self) -> PrefixCacheStats:
95 + """Get (and reset) the prefix cache stats.
96 +
97 + Returns:
98 + The current prefix caching stats.
99 + """
100 + stats = self.prefix_cache_stats
101 + self.prefix_cache_stats = PrefixCacheStats()
102 + return stats
103 +
104 def get_computed_blocks(
105 self, request: Request) -> Tuple[List[KVCacheBlock], int]:
106 """Get the computed (cached) blocks for the request.
@@ -118,6 +136,10 @@ class KVCacheManager:
136 else:
137 break
138
139 + self.prefix_cache_stats.requests += 1
140 + self.prefix_cache_stats.queries += len(block_hashes)
141 + self.prefix_cache_stats.hits += len(computed_blocks)
142 +
143 # NOTE(woosuk): Since incomplete blocks are not eligible for
144 # sharing, `num_computed_tokens` is always a multiple of
145 # `block_size`.
@@ -280,6 +302,8 @@ class KVCacheManager:
302 for block in self.block_pool:
303 block.reset_hash()
304
305 + self.prefix_cache_stats.reset = True
306 +
307 logger.info("Successfully reset prefix cache")
308 return True
309
vllm/v1/core/kv_cache_utils.py
+64
@@ -1,5 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 """KV-Cache Utilities."""
3 +from collections import deque
4 from collections.abc import Sequence
5 from dataclasses import dataclass
6 from typing import Any, List, NamedTuple, Optional, Tuple
@@ -8,6 +9,7 @@ from vllm.config import VllmConfig
9 from vllm.logger import init_logger
10 from vllm.v1.kv_cache_interface import (KVCacheConfig, KVCacheSpec,
11 KVCacheTensor)
12 +from vllm.v1.metrics.stats import PrefixCacheStats
13 from vllm.v1.request import Request
14
15 logger = init_logger(__name__)
@@ -28,6 +30,68 @@ class BlockHashType(NamedTuple):
30 extra_keys: Optional[Any] = None
31
32
33 +class PrefixCachingMetrics:
34 + """Metrics for prefix caching with a hit rate of the most recent N requests.
35 +
36 + Args:
37 + interval: The number of the most recent requests to aggregate.
38 + Defaults to 1000.
39 + """
40 +
41 + def __init__(self, interval: int = 1000):
42 + self.interval = interval
43 + # The current aggregated values.
44 + self.aggregated_requests = 0
45 + self.aggregated_query_total = 0
46 + self.aggregated_query_hit = 0
47 + # A deque of (requests, queries, hits) for the most recent requests.
48 + self.query_queue: deque[Tuple[int, int, int]] = deque()
49 +
50 + def observe(self, stats: PrefixCacheStats):
51 + """Observe the prefix caching for a set of requests.
52 +
53 + This function is called with information gathered when new requests
54 + are being scheduled and are looking for computed blocks.
55 +
56 + When there are more than `interval` requests, the oldest set of
57 + requestsare removed from the metrics.
58 +
59 + Args:
60 + stats: The prefix cache stats.
61 + """
62 + # reset_prefix_cache was invoked before the current update.
63 + # Reset the metrics before aggregating the current stats.
64 + if stats.reset:
65 + self.reset()
66 +
67 + # Update the metrics.
68 + self.query_queue.append((stats.requests, stats.queries, stats.hits))
69 + self.aggregated_requests += stats.requests
70 + self.aggregated_query_total += stats.queries
71 + self.aggregated_query_hit += stats.hits
72 +
73 + # Remove the oldest stats if the number of requests exceeds.
74 + if self.aggregated_requests > self.interval:
75 + old_requests, old_queries, old_hits = self.query_queue.popleft()
76 + self.aggregated_requests -= old_requests
77 + self.aggregated_query_total -= old_queries
78 + self.aggregated_query_hit -= old_hits
79 +
80 + def reset(self):
81 + """Reset the metrics."""
82 + self.aggregated_requests = 0
83 + self.aggregated_query_total = 0
84 + self.aggregated_query_hit = 0
85 + self.query_queue.clear()
86 +
87 + @property
88 + def hit_rate(self) -> float:
89 + """Calculate the hit rate for the past N requests."""
90 + if self.aggregated_query_total == 0:
91 + return 0.0
92 + return self.aggregated_query_hit / self.aggregated_query_total
93 +
94 +
95 @dataclass
96 class KVCacheBlock:
97 """KV-cache block metadata."""
vllm/v1/core/scheduler.py
+1
@@ -593,4 +593,5 @@ class Scheduler:
593 num_running_reqs=len(self.running),
594 num_waiting_reqs=len(self.waiting),
595 gpu_cache_usage=self.kv_cache_manager.usage,
596 + prefix_cache_stats=self.kv_cache_manager.make_prefix_cache_stats(),
597 )
vllm/v1/metrics/loggers.py
+27 -2
@@ -9,6 +9,7 @@ import prometheus_client
9
10 from vllm.config import ModelConfig
11 from vllm.logger import init_logger
12 +from vllm.v1.core.kv_cache_utils import PrefixCachingMetrics
13 from vllm.v1.engine import FinishReason
14 from vllm.v1.metrics.stats import IterationStats, SchedulerStats
15
@@ -37,6 +38,9 @@ class LoggingStatLogger(StatLoggerBase):
38 self.num_prompt_tokens: List[int] = []
39 self.num_generation_tokens: List[int] = []
40
41 + # Prefix cache metrics. TODO: Make the interval configurable.
42 + self.prefix_caching_metrics = PrefixCachingMetrics()
43 +
44 def _local_interval_elapsed(self, now: float) -> bool:
45 # Log every _LOCAL_LOGGING_INTERVAL_SEC.
46 elapsed_time = now - self.last_log_time
@@ -58,6 +62,8 @@ class LoggingStatLogger(StatLoggerBase):
62
63 self._track_iteration_stats(iteration_stats)
64
65 + self.prefix_caching_metrics.observe(scheduler_stats.prefix_cache_stats)
66 +
67 now = time.monotonic()
68 if not self._local_interval_elapsed(now):
69 return
@@ -72,13 +78,15 @@ class LoggingStatLogger(StatLoggerBase):
78 logger.info(
79 "Avg prompt throughput: %.1f tokens/s, "
80 "Avg generation throughput: %.1f tokens/s, "
75 - "Running: %d reqs, Waiting: %d reqs "
76 - "GPU KV cache usage: %.1f%%.",
81 + "Running: %d reqs, Waiting: %d reqs, "
82 + "GPU KV cache usage: %.1f%%, "
83 + "Prefix cache hit rate: %.1f%%",
84 prompt_throughput,
85 generation_throughput,
86 scheduler_stats.num_running_reqs,
87 scheduler_stats.num_waiting_reqs,
88 scheduler_stats.gpu_cache_usage * 100,
89 + self.prefix_caching_metrics.hit_rate * 100,
90 )
91
92
@@ -107,6 +115,18 @@ class PrometheusStatLogger(StatLoggerBase):
115 documentation="GPU KV-cache usage. 1 means 100 percent usage.",
116 labelnames=labelnames).labels(*labelvalues)
117
118 + self.counter_gpu_prefix_cache_queries = prometheus_client.Counter(
119 + name="vllm:gpu_prefix_cache_queries",
120 + documentation=
121 + "GPU prefix cache queries, in terms of number of queried blocks.",
122 + labelnames=labelnames).labels(*labelvalues)
123 +
124 + self.counter_gpu_prefix_cache_hits = prometheus_client.Counter(
125 + name="vllm:gpu_prefix_cache_hits",
126 + documentation=
127 + "GPU prefix cache hits, in terms of number of cached blocks.",
128 + labelnames=labelnames).labels(*labelvalues)
129 +
130 self.counter_prompt_tokens = prometheus_client.Counter(
131 name="vllm:prompt_tokens_total",
132 documentation="Number of prefill tokens processed.",
@@ -170,6 +190,11 @@ class PrometheusStatLogger(StatLoggerBase):
190
191 self.gauge_gpu_cache_usage.set(scheduler_stats.gpu_cache_usage)
192
193 + self.counter_gpu_prefix_cache_queries.inc(
194 + scheduler_stats.prefix_cache_stats.queries)
195 + self.counter_gpu_prefix_cache_hits.inc(
196 + scheduler_stats.prefix_cache_stats.hits)
197 +
198 self.counter_prompt_tokens.inc(iteration_stats.num_prompt_tokens)
199 self.counter_generation_tokens.inc(
200 iteration_stats.num_generation_tokens)
vllm/v1/metrics/stats.py
+18 -2
@@ -1,7 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import time
4 -from dataclasses import dataclass
4 +from dataclasses import dataclass, field
5 from typing import TYPE_CHECKING, List
6
7 if TYPE_CHECKING:
@@ -9,6 +9,20 @@ if TYPE_CHECKING:
9 from vllm.v1.engine import EngineCoreOutput, FinishReason
10
11
12 +@dataclass
13 +class PrefixCacheStats:
14 + """Stores prefix cache hit statistics."""
15 + # Whether reset_prefix_cache was invoked.
16 + reset: bool = False
17 + # The number of requests in this update.
18 + requests: int = 0
19 + # The number of queries in these requests. Note that "queries" here
20 + # means the number of blocks that were queried from the cache.
21 + queries: int = 0
22 + # The number of hits in these requests.
23 + hits: int = 0
24 +
25 +
26 @dataclass
27 class SchedulerStats:
28 """Stats associated with the scheduler."""
@@ -17,7 +31,9 @@ class SchedulerStats:
31 num_waiting_reqs: int = 0
32
33 gpu_cache_usage: float = 0.0
20 - # gpu_prefix_cache_hit_rate: float = 0.0
34 +
35 + prefix_cache_stats: PrefixCacheStats = field(
36 + default_factory=PrefixCacheStats)
37
38
39 @dataclass