A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[V1] Remove log noise when idle (#16735)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Apr 17, 2025 at 00:34 UTC
9dbf7a2dc1448d6657adfb2daba36be270dcebcd
2 files changed
+15
-4
vllm/v1/metrics/loggers.py
+13
-2
@@ -40,6 +40,8 @@ class LoggingStatLogger(StatLoggerBase):
40
# TODO: Make the interval configurable.
41
self.prefix_caching_metrics = PrefixCachingMetrics()
42
self.spec_decoding_metrics = SpecDecodingMetrics()
43
+ self.last_prompt_throughput: float = 0.0
44
+ self.last_generation_throughput: float = 0.0
45
46
def _reset(self, now):
47
self.last_log_time = now
@@ -83,8 +85,17 @@ class LoggingStatLogger(StatLoggerBase):
85
86
scheduler_stats = self.last_scheduler_stats
87
88
+ log_fn = logger.info
89
+ if not any(
90
+ (prompt_throughput, generation_throughput,
91
+ self.last_prompt_throughput, self.last_generation_throughput)):
92
+ # Avoid log noise on an idle production system
93
+ log_fn = logger.debug
94
+ self.last_generation_throughput = generation_throughput
95
+ self.last_prompt_throughput = prompt_throughput
96
+
97
# Format and print output.
87
- logger.info(
98
+ log_fn(
99
"Engine %03d: "
100
"Avg prompt throughput: %.1f tokens/s, "
101
"Avg generation throughput: %.1f tokens/s, "
@@ -101,7 +112,7 @@ class LoggingStatLogger(StatLoggerBase):
112
)
113
114
if scheduler_stats.spec_decoding_stats is not None:
104
- self.spec_decoding_metrics.log()
115
+ self.spec_decoding_metrics.log(log_fn=log_fn)
116
117
118
class PrometheusStatLogger(StatLoggerBase):
vllm/v1/spec_decode/metrics.py
+2
-2
@@ -43,14 +43,14 @@ class SpecDecodingMetrics:
43
self.num_accepted_tokens.append(
44
spec_decoding_stats.num_accepted_tokens)
45
46
- def log(self):
46
+ def log(self, log_fn=logger.info):
47
num_draft_tokens = np.sum(self.num_draft_tokens)
48
num_accepted_tokens = np.sum(self.num_accepted_tokens)
49
50
draft_acceptance_rate = (num_accepted_tokens / num_draft_tokens *
51
100 if num_draft_tokens > 0 else float("nan"))
52
53
- logger.info(
53
+ log_fn(
54
"SpecDecoding metrics: "
55
"Draft acceptance rate: %.1f%%, "
56
"Accepted: %d tokens, "