A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[core] Avoid metrics log noise when idle - include speculative decodi… (#10809)
cduk committed
Dec 2, 2024 at 02:49 UTC
b7954776fd338cab442a8004d240f7fe74e4e51b
1 file changed
+2
-2
vllm/engine/metrics.py
+2
-2
@@ -473,13 +473,13 @@ class LoggingStatLogger(StatLoggerBase):
473
)
474
if (stats.cpu_prefix_cache_hit_rate >= 0
475
or stats.gpu_prefix_cache_hit_rate >= 0):
476
- logger.info(
476
+ log_fn(
477
"Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
478
stats.gpu_prefix_cache_hit_rate * 100,
479
stats.cpu_prefix_cache_hit_rate * 100,
480
)
481
if self.spec_decode_metrics is not None:
482
- logger.info(
482
+ log_fn(
483
self._format_spec_decode_metrics_str(
484
self.spec_decode_metrics))
485