@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / efa9084628
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Core] Avoid metrics log noise when idle (#8868)

Signed-off-by: Russell Bryant <rbryant@redhat.com>

Russell Bryant committed Nov 19, 2024 at 16:05 UTC efa9084628b32787ae1901a2d1e9b80f7d08809b
1 file changed +23 -7
vllm/engine/metrics.py
+23 -7
@@ -421,6 +421,11 @@ def get_throughput(tracked_stats: List[int], now: float,
421 class LoggingStatLogger(StatLoggerBase):
422 """LoggingStatLogger is used in LLMEngine to log to Stdout."""
423
424 + def __init__(self, *args, **kwargs) -> None:
425 + super().__init__(*args, **kwargs)
426 + self.last_prompt_throughput: Optional[float] = None
427 + self.last_generation_throughput: Optional[float] = None
428 +
429 def log(self, stats: Stats) -> None:
430 """Called by LLMEngine.
431 Logs to Stdout every self.local_interval seconds."""
@@ -445,8 +450,14 @@ class LoggingStatLogger(StatLoggerBase):
450 now=stats.now,
451 last_log=self.last_local_log)
452
448 - # Log to stdout.
449 - logger.info(
453 + log_fn = logger.info
454 + if not any((prompt_throughput, generation_throughput,
455 + self.last_prompt_throughput,
456 + self.last_generation_throughput)):
457 + # Avoid log noise on an idle production system
458 + log_fn = logger.debug
459 +
460 + log_fn(
461 "Avg prompt throughput: %.1f tokens/s, "
462 "Avg generation throughput: %.1f tokens/s, "
463 "Running: %d reqs, Swapped: %d reqs, "
@@ -472,11 +483,16 @@ class LoggingStatLogger(StatLoggerBase):
483 self._format_spec_decode_metrics_str(
484 self.spec_decode_metrics))
485
475 - # Reset tracked stats for next interval.
476 - self.num_prompt_tokens = []
477 - self.num_generation_tokens = []
478 - self.last_local_log = stats.now
479 - self.spec_decode_metrics = None
486 + self._reset(stats, prompt_throughput, generation_throughput)
487 +
488 + def _reset(self, stats, prompt_throughput, generation_throughput) -> None:
489 + # Reset tracked stats for next interval.
490 + self.num_prompt_tokens = []
491 + self.num_generation_tokens = []
492 + self.last_local_log = stats.now
493 + self.spec_decode_metrics = None
494 + self.last_prompt_throughput = prompt_throughput
495 + self.last_generation_throughput = generation_throughput
496
497 def _format_spec_decode_metrics_str(
498 self, metrics: "SpecDecodeWorkerMetrics") -> str: