A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[V1][Metrics] Add per-request TPOT histogram (#24015)
Signed-off-by: baxingpiaochong <771405853@qq.com>
baxingpiaochong committed
Sep 24, 2025 at 08:19 UTC
d06b5a95cbd5114478ca87dabac3156a33bb6bca
2 files changed
+24
-1
vllm/v1/metrics/loggers.py
+15
@@ -411,6 +411,19 @@ class PrometheusStatLogger(StatLoggerBase):
411
self.histogram_inter_token_latency = make_per_engine(
412
histogram_inter_token_latency, engine_indexes, model_name)
413
414
+ histogram_request_time_per_output_token = self._histogram_cls(
415
+ name="vllm:request_time_per_output_token_seconds",
416
+ documentation=
417
+ "Histogram of time_per_output_token_seconds per request.",
418
+ buckets=[
419
+ 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
420
+ 1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0, 80.0
421
+ ],
422
+ labelnames=labelnames)
423
+ self.histogram_request_time_per_output_token = make_per_engine(
424
+ histogram_request_time_per_output_token, engine_indexes,
425
+ model_name)
426
+
427
request_latency_buckets = [
428
0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
429
40.0, 50.0, 60.0, 120.0, 240.0, 480.0, 960.0, 1920.0, 7680.0
@@ -583,6 +596,8 @@ class PrometheusStatLogger(StatLoggerBase):
596
finished_request.num_prompt_tokens)
597
self.histogram_num_generation_tokens_request[engine_idx].observe(
598
finished_request.num_generation_tokens)
599
+ self.histogram_request_time_per_output_token[engine_idx].observe(
600
+ finished_request.mean_time_per_output_token)
601
if finished_request.max_tokens_param:
602
self.histogram_max_tokens_request[engine_idx].observe(
603
finished_request.max_tokens_param)
vllm/v1/metrics/stats.py
+9
-1
@@ -86,6 +86,7 @@ class FinishedRequestStats:
86
prefill_time: float = 0.0
87
inference_time: float = 0.0
88
decode_time: float = 0.0
89
+ mean_time_per_output_token: float = 0.0
90
91
92
class IterationStats:
@@ -177,6 +178,12 @@ class IterationStats:
178
# Any preemptions during prefill or decode are included
179
inference_time = req_stats.last_token_ts - req_stats.scheduled_ts
180
181
+ # Do not count the token generated by the prefill phase
182
+ mean_time_per_output_token = (decode_time /
183
+ (req_stats.num_generation_tokens - 1)
184
+ if req_stats.num_generation_tokens -
185
+ 1 > 0 else 0)
186
+
187
finished_req = \
188
FinishedRequestStats(finish_reason=finish_reason,
189
e2e_latency=e2e_latency,
@@ -186,7 +193,8 @@ class IterationStats:
193
queued_time=queued_time,
194
prefill_time=prefill_time,
195
inference_time=inference_time,
189
- decode_time=decode_time)
196
+ decode_time=decode_time,
197
+ mean_time_per_output_token=mean_time_per_output_token)
198
self.finished_requests.append(finished_req)
199
200