@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 272e31c0bd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Guard for negative counter metrics to prevent crash (#10430)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Nov 18, 2024 at 21:57 UTC 272e31c0bd8640c15e85211c74fc9b428ad86902
2 files changed +6 -1
vllm/engine/llm_engine.py
+1 -1
@@ -1716,7 +1716,7 @@ class LLMEngine:
1716 # not counted (to avoid double counting)
1717 actual_num_batched_tokens = scheduler_outputs.num_batched_tokens # type: ignore
1718
1719 - num_generation_tokens_from_prefill_groups = 0.
1719 + num_generation_tokens_from_prefill_groups = 0
1720 # NOTE: if scheduler_outputs.num_prefill_groups > 0 and
1721 # the len of scheduler_outputs.scheduled_seq_groups is !=
1722 # scheduler_outputs.num_prefill_groups, this means that
vllm/engine/metrics.py
+5
@@ -512,6 +512,11 @@ class PrometheusStatLogger(StatLoggerBase):
512
513 def _log_counter(self, counter, data: Union[int, float]) -> None:
514 # Convenience function for logging to counter.
515 + # Prevent ValueError from negative increment
516 + if data < 0:
517 + logger.warning("Skipping negative increment of %g to %s", data,
518 + counter)
519 + return
520 counter.labels(**self.labels).inc(data)
521
522 def _log_counter_labels(self, counter, data: CollectionsCounter,