A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Core][Prefix Hash] Fix prefix hash metrics sliding window maintainance (#24990)
Signed-off-by: Jialin Ouyang <Jialin.Ouyang@gmail.com>
Jialin Ouyang committed
Sep 19, 2025 at 11:22 UTC
2506ce5189fa67fea905c1f18ca735191cec0d29
2 files changed
+50
-9
tests/v1/core/test_kv_cache_utils.py
+39
-7
@@ -513,27 +513,27 @@ def test_hash_request_tokens_no_mm_inputs(hash_fn):
513
assert block_hashes[1] == hash_fn((block_hashes[0], (3, 4, 5), None))
514
515
516
+def _stats(requests: int, queries: int, hits: int) -> PrefixCacheStats:
517
+ return PrefixCacheStats(requests=requests, queries=queries, hits=hits)
518
+
519
+
520
def test_metrics():
521
"""
522
Test the prefix caching metrics.
523
"""
520
-
521
- def stats(requests, queries, hits):
522
- return PrefixCacheStats(requests=requests, queries=queries, hits=hits)
523
-
524
metrics = PrefixCachingMetrics(max_recent_requests=5)
525
assert metrics.hit_rate == 0.0
526
527
- metrics.observe(stats(1, 20, 9))
527
+ metrics.observe(_stats(1, 20, 9))
528
# 9 / 20 = 0.45
529
assert metrics.hit_rate == 0.45
530
531
- metrics.observe(stats(4, 80, 16))
531
+ metrics.observe(_stats(4, 80, 16))
532
533
# 25 / 100 = 0.25
534
assert metrics.hit_rate == 0.25
535
536
- metrics.observe(stats(1, 10, 2))
536
+ metrics.observe(_stats(1, 10, 2))
537
538
# Remove (20, 9) and add (10, 2): 18 / 90 = 0.2
539
assert metrics.aggregated_requests == 5
@@ -549,6 +549,38 @@ def test_metrics():
549
assert not metrics.query_queue
550
551
552
+def test_metrics_empty_stats():
553
+ """
554
+ Test the prefix caching metrics with empty stats.
555
+ """
556
+ metrics = PrefixCachingMetrics(max_recent_requests=5)
557
+ metrics.observe(_stats(0, 0, 0))
558
+ metrics.observe(_stats(1, 20, 9))
559
+ metrics.observe(_stats(0, 0, 0))
560
+ metrics.observe(_stats(4, 80, 16))
561
+ metrics.observe(_stats(0, 0, 0))
562
+ metrics.observe(_stats(1, 10, 2))
563
+ # Remove (20, 9) and add (10, 2): 18 / 90 = 0.2
564
+ assert metrics.aggregated_requests == 5
565
+ assert metrics.aggregated_query_total == 90
566
+ assert metrics.aggregated_query_hit == 18
567
+ assert metrics.hit_rate == 0.2
568
+
569
+ # Only the latest added stats preserved 10 / 20 = 0.5
570
+ metrics.observe(_stats(11, 20, 10))
571
+ assert metrics.aggregated_requests == 11
572
+ assert metrics.aggregated_query_total == 20
573
+ assert metrics.aggregated_query_hit == 10
574
+ assert metrics.hit_rate == 0.5
575
+
576
+ # Only the latest added stats preserved 30 / 40 = 0.75
577
+ metrics.observe(_stats(22, 40, 30))
578
+ assert metrics.aggregated_requests == 22
579
+ assert metrics.aggregated_query_total == 40
580
+ assert metrics.aggregated_query_hit == 30
581
+ assert metrics.hit_rate == 0.75
582
+
583
+
584
def test_get_kv_cache_configs_multiple_workers():
585
model_config = ModelConfig(max_model_len=16)
586
vllm_config = VllmConfig(model_config=model_config)
vllm/v1/core/kv_cache_utils.py
+11
-2
@@ -127,14 +127,23 @@ class PrefixCachingMetrics:
127
if stats.reset:
128
self.reset()
129
130
+ # DO NOT appending empty stats to avoid helpful info get kicked out
131
+ # due to sliding window.
132
+ if stats.requests == 0:
133
+ return
134
+
135
# Update the metrics.
136
self.query_queue.append((stats.requests, stats.queries, stats.hits))
137
self.aggregated_requests += stats.requests
138
self.aggregated_query_total += stats.queries
139
self.aggregated_query_hit += stats.hits
140
136
- # Remove the oldest stats if the number of requests exceeds.
137
- if self.aggregated_requests > self.max_recent_requests:
141
+ # Remove the oldest stats until number of requests does not exceed
142
+ # the limit.
143
+ # NOTE: We preserve the latest added stats regardless.
144
+ while len(
145
+ self.query_queue
146
+ ) > 1 and self.aggregated_requests > self.max_recent_requests:
147
old_requests, old_queries, old_hits = self.query_queue.popleft()
148
self.aggregated_requests -= old_requests
149
self.aggregated_query_total -= old_queries