A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[MISC] rename interval to max_recent_requests (#14285)
Ning Xie committed
Apr 27, 2025 at 00:59 UTC
fd11a325b8dc1db54c51ac4ca10e0c9fac5e4298
2 files changed
+7
-7
tests/v1/core/test_kv_cache_utils.py
+1
-1
@@ -310,7 +310,7 @@ def test_metrics():
310
def stats(requests, queries, hits):
311
return PrefixCacheStats(requests=requests, queries=queries, hits=hits)
312
313
- metrics = PrefixCachingMetrics(interval=5)
313
+ metrics = PrefixCachingMetrics(max_recent_requests=5)
314
assert metrics.hit_rate == 0.0
315
316
metrics.observe(stats(1, 20, 9))
vllm/v1/core/kv_cache_utils.py
+6
-6
@@ -47,15 +47,15 @@ NONE_HASH = int.from_bytes(os.urandom(32), byteorder="big") if os.getenv(
47
48
49
class PrefixCachingMetrics:
50
- """Metrics for prefix caching with a hit rate of the most recent N requests.
50
+ """Metrics for prefix caching with a hit rate of the max recent N requests.
51
52
Args:
53
- interval: The number of the most recent requests to aggregate.
53
+ max_recent_requests: The number of the max recent requests to aggregate.
54
Defaults to 1000.
55
"""
56
57
- def __init__(self, interval: int = 1000):
58
- self.interval = interval
57
+ def __init__(self, max_recent_requests: int = 1000):
58
+ self.max_recent_requests = max_recent_requests
59
# The current aggregated values.
60
self.aggregated_requests = 0
61
self.aggregated_query_total = 0
@@ -70,7 +70,7 @@ class PrefixCachingMetrics:
70
are being scheduled and are looking for computed blocks.
71
72
When there are more than `interval` requests, the oldest set of
73
- requestsare removed from the metrics.
73
+ requests are removed from the metrics.
74
75
Args:
76
stats: The prefix cache stats.
@@ -87,7 +87,7 @@ class PrefixCachingMetrics:
87
self.aggregated_query_hit += stats.hits
88
89
# Remove the oldest stats if the number of requests exceeds.
90
- if self.aggregated_requests > self.interval:
90
+ if self.aggregated_requests > self.max_recent_requests:
91
old_requests, old_queries, old_hits = self.query_queue.popleft()
92
self.aggregated_requests -= old_requests
93
self.aggregated_query_total -= old_queries