A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Core] Avoid metrics log noise when idle (#8868)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Nov 19, 2024 at 16:05 UTC
efa9084628b32787ae1901a2d1e9b80f7d08809b
1 file changed
+23
-7
vllm/engine/metrics.py
+23
-7
@@ -421,6 +421,11 @@ def get_throughput(tracked_stats: List[int], now: float,
421
class LoggingStatLogger(StatLoggerBase):
422
"""LoggingStatLogger is used in LLMEngine to log to Stdout."""
423
424
+ def __init__(self, *args, **kwargs) -> None:
425
+ super().__init__(*args, **kwargs)
426
+ self.last_prompt_throughput: Optional[float] = None
427
+ self.last_generation_throughput: Optional[float] = None
428
+
429
def log(self, stats: Stats) -> None:
430
"""Called by LLMEngine.
431
Logs to Stdout every self.local_interval seconds."""
@@ -445,8 +450,14 @@ class LoggingStatLogger(StatLoggerBase):
450
now=stats.now,
451
last_log=self.last_local_log)
452
448
- # Log to stdout.
449
- logger.info(
453
+ log_fn = logger.info
454
+ if not any((prompt_throughput, generation_throughput,
455
+ self.last_prompt_throughput,
456
+ self.last_generation_throughput)):
457
+ # Avoid log noise on an idle production system
458
+ log_fn = logger.debug
459
+
460
+ log_fn(
461
"Avg prompt throughput: %.1f tokens/s, "
462
"Avg generation throughput: %.1f tokens/s, "
463
"Running: %d reqs, Swapped: %d reqs, "
@@ -472,11 +483,16 @@ class LoggingStatLogger(StatLoggerBase):
483
self._format_spec_decode_metrics_str(
484
self.spec_decode_metrics))
485
475
- # Reset tracked stats for next interval.
476
- self.num_prompt_tokens = []
477
- self.num_generation_tokens = []
478
- self.last_local_log = stats.now
479
- self.spec_decode_metrics = None
486
+ self._reset(stats, prompt_throughput, generation_throughput)
487
+
488
+ def _reset(self, stats, prompt_throughput, generation_throughput) -> None:
489
+ # Reset tracked stats for next interval.
490
+ self.num_prompt_tokens = []
491
+ self.num_generation_tokens = []
492
+ self.last_local_log = stats.now
493
+ self.spec_decode_metrics = None
494
+ self.last_prompt_throughput = prompt_throughput
495
+ self.last_generation_throughput = generation_throughput
496
497
def _format_spec_decode_metrics_str(
498
self, metrics: "SpecDecodeWorkerMetrics") -> str: