A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[v1][KVCacheManager] Change prefix caching metric from counting blocks to counting tokens (#18003)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
May 13, 2025 at 04:46 UTC
302f3aca7ea3f57842881cb2ae0062c19ad24758
3 files changed
+9
-9
vllm/v1/core/kv_cache_manager.py
+6
-6
@@ -161,11 +161,15 @@ class KVCacheManager:
161
162
computed_blocks = (
163
self.single_type_manager.find_longest_cache_hit(block_hashes))
164
+ # NOTE(woosuk): Since incomplete blocks are not eligible for
165
+ # sharing, `num_computed_tokens` is always a multiple of
166
+ # `block_size`.
167
+ num_computed_tokens = len(computed_blocks) * self.block_size
168
169
if self.log_stats:
170
assert self.prefix_cache_stats is not None
167
- self.prefix_cache_stats.queries += len(block_hashes)
168
- self.prefix_cache_stats.hits += len(computed_blocks)
171
+ self.prefix_cache_stats.queries += request.num_tokens
172
+ self.prefix_cache_stats.hits += num_computed_tokens
173
174
if last_block_hash is not None:
175
# Add back the last block hash if it was removed.
@@ -173,10 +177,6 @@ class KVCacheManager:
177
# we shouldn't modify it directly.
178
block_hashes.append(last_block_hash)
179
176
- # NOTE(woosuk): Since incomplete blocks are not eligible for
177
- # sharing, `num_computed_tokens` is always a multiple of
178
- # `block_size`.
179
- num_computed_tokens = len(computed_blocks) * self.block_size
180
return KVCacheBlocks(computed_blocks), num_computed_tokens
181
182
def allocate_slots(
vllm/v1/metrics/loggers.py
+2
-2
@@ -183,13 +183,13 @@ class PrometheusStatLogger(StatLoggerBase):
183
self.counter_gpu_prefix_cache_queries = prometheus_client.Counter(
184
name="vllm:gpu_prefix_cache_queries",
185
documentation=
186
- "GPU prefix cache queries, in terms of number of queried blocks.",
186
+ "GPU prefix cache queries, in terms of number of queried tokens.",
187
labelnames=labelnames).labels(*labelvalues)
188
189
self.counter_gpu_prefix_cache_hits = prometheus_client.Counter(
190
name="vllm:gpu_prefix_cache_hits",
191
documentation=
192
- "GPU prefix cache hits, in terms of number of cached blocks.",
192
+ "GPU prefix cache hits, in terms of number of cached tokens.",
193
labelnames=labelnames).labels(*labelvalues)
194
195
#
vllm/v1/metrics/stats.py
+1
-1
@@ -19,7 +19,7 @@ class PrefixCacheStats:
19
# The number of requests in this update.
20
requests: int = 0
21
# The number of queries in these requests. Note that "queries" here
22
- # means the number of blocks that were queried from the cache.
22
+ # means the number of tokens that were queried from the cache.
23
queries: int = 0
24
# The number of hits in these requests.
25
hits: int = 0