A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[V1][Bugfix] Do not reset prefix caching metrics (#14235)
Cody Yu committed
Mar 4, 2025 at 20:39 UTC
ade3f7d98869da44396d21d1c091be7edd3e1c8e
1 file changed
+3
-3
vllm/v1/metrics/loggers.py
+3
-3
@@ -34,6 +34,9 @@ class LoggingStatLogger(StatLoggerBase):
34
def __init__(self):
35
self._reset(time.monotonic())
36
self.last_scheduler_stats = SchedulerStats()
37
+ # Prefix cache metrics. This cannot be reset.
38
+ # TODO: Make the interval configurable.
39
+ self.prefix_caching_metrics = PrefixCachingMetrics()
40
41
def _reset(self, now):
42
self.last_log_time = now
@@ -42,9 +45,6 @@ class LoggingStatLogger(StatLoggerBase):
45
self.num_prompt_tokens: list[int] = []
46
self.num_generation_tokens: list[int] = []
47
45
- # Prefix cache metrics. TODO: Make the interval configurable.
46
- self.prefix_caching_metrics = PrefixCachingMetrics()
47
-
48
def _track_iteration_stats(self, iteration_stats: IterationStats):
49
# Save tracked stats for token counters.
50
self.num_prompt_tokens.append(iteration_stats.num_prompt_tokens)