A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Frontend] correctly record prefill and decode time metrics (#10853)
Signed-off-by: Tomer Asida <tomera@ai21.com>
tomeras91 committed
Dec 3, 2024 at 21:13 UTC
7c32b6861e20b6521959b6cc1ce7ccc84614974d
1 file changed
+2
-2
vllm/engine/metrics.py
+2
-2
@@ -599,9 +599,9 @@ class PrometheusStatLogger(StatLoggerBase):
599
stats.time_queue_requests)
600
self._log_histogram(self.metrics.histogram_inference_time_request,
601
stats.time_inference_requests)
602
- self._log_histogram(self.metrics.histogram_decode_time_request,
603
- stats.time_prefill_requests)
602
self._log_histogram(self.metrics.histogram_prefill_time_request,
603
+ stats.time_prefill_requests)
604
+ self._log_histogram(self.metrics.histogram_decode_time_request,
605
stats.time_decode_requests)
606
self._log_histogram(self.metrics.histogram_time_in_queue_request,
607
stats.time_in_queue_requests)