A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[V1][Spec Decoding] Log accumulated metrics after system goes idle (#17913)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
May 10, 2025 at 02:23 UTC
7042cc96b0a8a154ea165c652d4f63e5be9c291e
2 files changed
+3
-3
vllm/v1/metrics/loggers.py
+1
-3
@@ -128,9 +128,7 @@ class LoggingStatLogger(StatLoggerBase):
128
scheduler_stats.gpu_cache_usage * 100,
129
self.prefix_caching_metrics.hit_rate * 100,
130
)
131
-
132
- if scheduler_stats.spec_decoding_stats is not None:
133
- self.spec_decoding_logging.log(log_fn=log_fn)
131
+ self.spec_decoding_logging.log(log_fn=log_fn)
132
133
def log_engine_initialized(self):
134
logger.info(
vllm/v1/spec_decode/metrics.py
+2
@@ -67,6 +67,8 @@ class SpecDecodingLogging:
67
spec_decoding_stats.num_accepted_tokens_per_pos)
68
69
def log(self, log_fn=logger.info):
70
+ if not self.num_drafts:
71
+ return
72
num_drafts = np.sum(self.num_drafts)
73
num_draft_tokens = np.sum(self.num_draft_tokens)
74
num_accepted_tokens = np.sum(self.num_accepted_tokens)