A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Metrics] Deprecate TPOT in favor of ITL (#24110)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
Sep 2, 2025 at 19:10 UTC
2417798471af8521e488c04f3a43c91e6836a705
7 files changed
+64
-26
examples/online_serving/prometheus_grafana/grafana.json
+6
-6
@@ -402,7 +402,7 @@
402
},
403
"disableTextWrap": false,
404
"editorMode": "builder",
405
- "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
405
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
406
"fullMetaSearch": false,
407
"includeNullMetadata": false,
408
"instant": false,
@@ -418,7 +418,7 @@
418
},
419
"disableTextWrap": false,
420
"editorMode": "builder",
421
- "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
421
+ "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
422
"fullMetaSearch": false,
423
"hide": false,
424
"includeNullMetadata": false,
@@ -435,7 +435,7 @@
435
},
436
"disableTextWrap": false,
437
"editorMode": "builder",
438
- "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
438
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
439
"fullMetaSearch": false,
440
"hide": false,
441
"includeNullMetadata": false,
@@ -452,7 +452,7 @@
452
},
453
"disableTextWrap": false,
454
"editorMode": "builder",
455
- "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
455
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
456
"fullMetaSearch": false,
457
"hide": false,
458
"includeNullMetadata": false,
@@ -468,7 +468,7 @@
468
"uid": "${DS_PROMETHEUS}"
469
},
470
"editorMode": "code",
471
- "expr": "rate(vllm:time_per_output_token_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:time_per_output_token_seconds_count{model_name=\"$model_name\"}[$__rate_interval])",
471
+ "expr": "rate(vllm:inter_token_latency_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:inter_token_latency_seconds_count{model_name=\"$model_name\"}[$__rate_interval])",
472
"hide": false,
473
"instant": false,
474
"legendFormat": "Mean",
@@ -476,7 +476,7 @@
476
"refId": "E"
477
}
478
],
479
- "title": "Time Per Output Token Latency",
479
+ "title": "Inter Token Latency",
480
"type": "timeseries"
481
},
482
{
tests/entrypoints/openai/test_metrics.py
+15
-7
@@ -250,12 +250,15 @@ EXPECTED_METRICS_V1 = [
250
"vllm:request_params_max_tokens_sum",
251
"vllm:request_params_max_tokens_bucket",
252
"vllm:request_params_max_tokens_count",
253
- "vllm:time_to_first_token_seconds_sum",
254
- "vllm:time_to_first_token_seconds_bucket",
255
- "vllm:time_to_first_token_seconds_count",
253
"vllm:time_per_output_token_seconds_sum",
254
"vllm:time_per_output_token_seconds_bucket",
255
"vllm:time_per_output_token_seconds_count",
256
+ "vllm:time_to_first_token_seconds_sum",
257
+ "vllm:time_to_first_token_seconds_bucket",
258
+ "vllm:time_to_first_token_seconds_count",
259
+ "vllm:inter_token_latency_seconds_sum",
260
+ "vllm:inter_token_latency_seconds_bucket",
261
+ "vllm:inter_token_latency_seconds_count",
262
"vllm:e2e_request_latency_seconds_sum",
263
"vllm:e2e_request_latency_seconds_bucket",
264
"vllm:e2e_request_latency_seconds_count",
@@ -273,7 +276,11 @@ EXPECTED_METRICS_V1 = [
276
"vllm:request_decode_time_seconds_count",
277
]
278
276
-HIDDEN_DEPRECATED_METRICS: list[str] = []
279
+HIDDEN_DEPRECATED_METRICS: list[str] = [
280
+ "vllm:time_per_output_token_seconds_sum",
281
+ "vllm:time_per_output_token_seconds_bucket",
282
+ "vllm:time_per_output_token_seconds_count",
283
+]
284
285
286
@pytest.mark.asyncio
@@ -289,9 +296,10 @@ async def test_metrics_exist(server: RemoteOpenAIServer,
296
assert response.status_code == HTTPStatus.OK
297
298
for metric in (EXPECTED_METRICS_V1 if use_v1 else EXPECTED_METRICS):
292
- if (not server.show_hidden_metrics
293
- and metric not in HIDDEN_DEPRECATED_METRICS):
294
- assert metric in response.text
299
+ if (metric in HIDDEN_DEPRECATED_METRICS
300
+ and not server.show_hidden_metrics):
301
+ continue
302
+ assert metric in response.text
303
304
305
@pytest.mark.asyncio
vllm/engine/llm_engine.py
+4
-4
@@ -1414,7 +1414,7 @@ class LLMEngine:
1414
num_generation_tokens_iter = 0
1415
num_tokens_iter = 0
1416
time_to_first_tokens_iter: List[float] = []
1417
- time_per_output_tokens_iter: List[float] = []
1417
+ inter_token_latencies_iter: List[float] = []
1418
num_preemption_iter = (0 if scheduler_outputs is None else
1419
scheduler_outputs.preempted)
1420
@@ -1498,9 +1498,9 @@ class LLMEngine:
1498
num_generation_tokens_from_prefill_groups += (
1499
seq_group.num_seqs())
1500
else:
1501
- # TPOTs.
1501
+ # ITLs
1502
latency = seq_group.get_last_token_latency()
1503
- time_per_output_tokens_iter.append(latency)
1503
+ inter_token_latencies_iter.append(latency)
1504
if seq_group.state.current_step == 0:
1505
# For async_output_proc, the do_log_stats()
1506
# is called following init_multi_step(), which
@@ -1582,7 +1582,7 @@ class LLMEngine:
1582
num_generation_tokens_iter=num_generation_tokens_iter,
1583
num_tokens_iter=num_tokens_iter,
1584
time_to_first_tokens_iter=time_to_first_tokens_iter,
1585
- time_per_output_tokens_iter=time_per_output_tokens_iter,
1585
+ inter_token_latencies_iter=inter_token_latencies_iter,
1586
num_preemption_iter=num_preemption_iter,
1587
1588
# Request stats
vllm/engine/metrics.py
+16
-2
@@ -113,9 +113,21 @@ class Metrics:
113
0.75, 1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0, 80.0, 160.0, 640.0,
114
2560.0
115
])
116
+ # Deprecated in 0.11 - Renamed as vllm:inter_token_latency_seconds
117
+ # TODO: in 0.12, only enable if show_hidden_metrics=True
118
self.histogram_time_per_output_token = self._histogram_cls(
119
name="vllm:time_per_output_token_seconds",
118
- documentation="Histogram of time per output token in seconds.",
120
+ documentation=(
121
+ "Histogram of time per output token in seconds."
122
+ "DEPRECATED: Use vllm:inter_token_latency_seconds instead."),
123
+ labelnames=labelnames,
124
+ buckets=[
125
+ 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
126
+ 1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0, 80.0
127
+ ])
128
+ self.histogram_inter_token_latency = self._histogram_cls(
129
+ name="vllm:inter_token_latency_seconds",
130
+ documentation="Histogram of inter token latency in seconds.",
131
labelnames=labelnames,
132
buckets=[
133
0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
@@ -491,7 +503,9 @@ class PrometheusStatLogger(StatLoggerBase):
503
self._log_histogram(self.metrics.histogram_time_to_first_token,
504
stats.time_to_first_tokens_iter)
505
self._log_histogram(self.metrics.histogram_time_per_output_token,
494
- stats.time_per_output_tokens_iter)
506
+ stats.inter_token_latencies_iter)
507
+ self._log_histogram(self.metrics.histogram_inter_token_latency,
508
+ stats.inter_token_latencies_iter)
509
510
# Request level data
511
# Latency
vllm/engine/metrics_types.py
+1
-1
@@ -43,7 +43,7 @@ class Stats:
43
num_generation_tokens_iter: int
44
num_tokens_iter: int
45
time_to_first_tokens_iter: List[float]
46
- time_per_output_tokens_iter: List[float]
46
+ inter_token_latencies_iter: List[float]
47
num_preemption_iter: int
48
49
# Request stats (should have _requests suffix)
vllm/v1/metrics/loggers.py
+19
-3
@@ -377,9 +377,13 @@ class PrometheusStatLogger(StatLoggerBase):
377
self.histogram_time_to_first_token = make_per_engine(
378
histogram_time_to_first_token, engine_indexes, model_name)
379
380
+ # Deprecated in 0.11 - Renamed as vllm:inter_token_latency_seconds
381
+ # TODO: in 0.12, only enable if show_hidden_metrics=True
382
histogram_time_per_output_token = self._histogram_cls(
383
name="vllm:time_per_output_token_seconds",
382
- documentation="Histogram of time per output token in seconds.",
384
+ documentation=(
385
+ "Histogram of time per output token in seconds."
386
+ "DEPRECATED: Use vllm:inter_token_latency_seconds instead."),
387
buckets=[
388
0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
389
1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0, 80.0
@@ -388,6 +392,17 @@ class PrometheusStatLogger(StatLoggerBase):
392
self.histogram_time_per_output_token = make_per_engine(
393
histogram_time_per_output_token, engine_indexes, model_name)
394
395
+ histogram_inter_token_latency = self._histogram_cls(
396
+ name="vllm:inter_token_latency_seconds",
397
+ documentation="Histogram of inter-token latency in seconds.",
398
+ buckets=[
399
+ 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
400
+ 1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0, 80.0
401
+ ],
402
+ labelnames=labelnames)
403
+ self.histogram_inter_token_latency = make_per_engine(
404
+ histogram_inter_token_latency, engine_indexes, model_name)
405
+
406
request_latency_buckets = [
407
0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
408
40.0, 50.0, 60.0, 120.0, 240.0, 480.0, 960.0, 1920.0, 7680.0
@@ -537,8 +552,9 @@ class PrometheusStatLogger(StatLoggerBase):
552
self.histogram_n_request[engine_idx].observe(n_param)
553
for ttft in iteration_stats.time_to_first_tokens_iter:
554
self.histogram_time_to_first_token[engine_idx].observe(ttft)
540
- for tpot in iteration_stats.time_per_output_tokens_iter:
541
- self.histogram_time_per_output_token[engine_idx].observe(tpot)
555
+ for itl in iteration_stats.inter_token_latencies_iter:
556
+ self.histogram_inter_token_latency[engine_idx].observe(itl)
557
+ self.histogram_time_per_output_token[engine_idx].observe(itl)
558
559
for finished_request in iteration_stats.finished_requests:
560
self.counter_request_success[
vllm/v1/metrics/stats.py
+3
-3
@@ -96,7 +96,7 @@ class IterationStats:
96
self.max_num_generation_tokens_iter: list[int] = []
97
self.n_params_iter: list[int] = []
98
self.time_to_first_tokens_iter: list[float] = []
99
- self.time_per_output_tokens_iter: list[float] = []
99
+ self.inter_token_latencies_iter: list[float] = []
100
self.waiting_lora_adapters: dict[str, int] = {}
101
self.running_lora_adapters: dict[str, int] = {}
102
@@ -128,8 +128,8 @@ class IterationStats:
128
if is_prefilling:
129
req_stats.first_token_ts = engine_core_timestamp
130
else:
131
- tpot = engine_core_timestamp - req_stats.last_token_ts
132
- self.time_per_output_tokens_iter.append(tpot)
131
+ itl = engine_core_timestamp - req_stats.last_token_ts
132
+ self.inter_token_latencies_iter.append(itl)
133
134
req_stats.last_token_ts = engine_core_timestamp
135