A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] Guard for negative counter metrics to prevent crash (#10430)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Nov 18, 2024 at 21:57 UTC
272e31c0bd8640c15e85211c74fc9b428ad86902
2 files changed
+6
-1
vllm/engine/llm_engine.py
+1
-1
@@ -1716,7 +1716,7 @@ class LLMEngine:
1716
# not counted (to avoid double counting)
1717
actual_num_batched_tokens = scheduler_outputs.num_batched_tokens # type: ignore
1718
1719
- num_generation_tokens_from_prefill_groups = 0.
1719
+ num_generation_tokens_from_prefill_groups = 0
1720
# NOTE: if scheduler_outputs.num_prefill_groups > 0 and
1721
# the len of scheduler_outputs.scheduled_seq_groups is !=
1722
# scheduler_outputs.num_prefill_groups, this means that
vllm/engine/metrics.py
+5
@@ -512,6 +512,11 @@ class PrometheusStatLogger(StatLoggerBase):
512
513
def _log_counter(self, counter, data: Union[int, float]) -> None:
514
# Convenience function for logging to counter.
515
+ # Prevent ValueError from negative increment
516
+ if data < 0:
517
+ logger.warning("Skipping negative increment of %g to %s", data,
518
+ counter)
519
+ return
520
counter.labels(**self.labels).inc(data)
521
522
def _log_counter_labels(self, counter, data: CollectionsCounter,