@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b7954776fd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[core] Avoid metrics log noise when idle - include speculative decodi… (#10809)

cduk committed Dec 2, 2024 at 02:49 UTC b7954776fd338cab442a8004d240f7fe74e4e51b
1 file changed +2 -2
vllm/engine/metrics.py
+2 -2
@@ -473,13 +473,13 @@ class LoggingStatLogger(StatLoggerBase):
473 )
474 if (stats.cpu_prefix_cache_hit_rate >= 0
475 or stats.gpu_prefix_cache_hit_rate >= 0):
476 - logger.info(
476 + log_fn(
477 "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
478 stats.gpu_prefix_cache_hit_rate * 100,
479 stats.cpu_prefix_cache_hit_rate * 100,
480 )
481 if self.spec_decode_metrics is not None:
482 - logger.info(
482 + log_fn(
483 self._format_spec_decode_metrics_str(
484 self.spec_decode_metrics))
485