A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 23:00 UTC
[Metrics] Complete removal of deprecated vllm:time_per_output_token_seconds metric (#32661)
This PR completes the removal of the deprecated vllm:time_per_output_token_seconds metric that was deprecated in v0.11, hidden in v0.12, scheduled for removal in v0.13, but delayed until v0.15. Signed-off-by: carlory <baofa.fan@daocloud.io> Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
杨朱 · Kiki committed
Jan 20, 2026 at 20:28 UTC
bb9172030e302300499e74c6662c6effdcdd8f66
5 files changed
+20
-65
docs/design/metrics.md
+1
-1
@@ -49,7 +49,7 @@ The subset of metrics exposed in the Grafana dashboard gives us an indication of
49
- `vllm:e2e_request_latency_seconds_bucket` - End to end request latency measured in seconds.
50
- `vllm:prompt_tokens` - Prompt tokens.
51
- `vllm:generation_tokens` - Generation tokens.
52
-- `vllm:time_per_output_token_seconds` - Inter-token latency (Time Per Output Token, TPOT) in seconds.
52
+- `vllm:inter_token_latency_seconds` - Inter-token latency (Time Per Output Token, TPOT) in seconds.
53
- `vllm:time_to_first_token_seconds` - Time to First Token (TTFT) latency in seconds.
54
- `vllm:num_requests_running` (also, `_swapped` and `_waiting`) - Number of requests in the RUNNING, WAITING, and SWAPPED states.
55
- `vllm:kv_cache_usage_perc` - Percentage of used cache blocks by vLLM.
examples/online_serving/dashboards/grafana/performance_statistics.json
+8
-8
@@ -884,7 +884,7 @@
884
"targets": [
885
{
886
"editorMode": "code",
887
- "expr": "rate(vllm:time_per_output_token_seconds_sum[$__interval]) / rate(vllm:time_per_output_token_seconds_count[$__interval])",
887
+ "expr": "rate(vllm:inter_token_latency_seconds_sum[$__interval]) / rate(vllm:inter_token_latency_seconds_count[$__interval])",
888
"legendFormat": "ITL (Avg)",
889
"range": true,
890
"refId": "A"
@@ -895,7 +895,7 @@
895
"uid": "${DS_PROMETHEUS}"
896
},
897
"editorMode": "code",
898
- "expr": "histogram_quantile(0.50, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__interval])))",
898
+ "expr": "histogram_quantile(0.50, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket[$__interval])))",
899
"hide": false,
900
"instant": false,
901
"legendFormat": "ITL (p50)",
@@ -908,7 +908,7 @@
908
"uid": "${DS_PROMETHEUS}"
909
},
910
"editorMode": "code",
911
- "expr": "histogram_quantile(0.90, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__interval])))",
911
+ "expr": "histogram_quantile(0.90, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket[$__interval])))",
912
"hide": false,
913
"instant": false,
914
"legendFormat": "ITL (p90)",
@@ -921,7 +921,7 @@
921
"uid": "${DS_PROMETHEUS}"
922
},
923
"editorMode": "code",
924
- "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__interval])))",
924
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket[$__interval])))",
925
"hide": false,
926
"instant": false,
927
"legendFormat": "ITL (p99)",
@@ -990,7 +990,7 @@
990
"targets": [
991
{
992
"editorMode": "code",
993
- "expr": "histogram_quantile(0.90, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__range])))",
993
+ "expr": "histogram_quantile(0.90, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket[$__range])))",
994
"legendFormat": "__auto",
995
"range": true,
996
"refId": "A"
@@ -1057,7 +1057,7 @@
1057
"targets": [
1058
{
1059
"editorMode": "code",
1060
- "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__range])))",
1060
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket[$__range])))",
1061
"legendFormat": "__auto",
1062
"range": true,
1063
"refId": "A"
@@ -1124,7 +1124,7 @@
1124
"targets": [
1125
{
1126
"editorMode": "code",
1127
- "expr": "(sum(increase(vllm:time_per_output_token_seconds_sum[$__range])) / sum(increase(vllm:time_per_output_token_seconds_count[$__range])))",
1127
+ "expr": "(sum(increase(vllm:inter_token_latency_seconds_sum[$__range])) / sum(increase(vllm:inter_token_latency_seconds_count[$__range])))",
1128
"legendFormat": "__auto",
1129
"range": true,
1130
"refId": "A"
@@ -1191,7 +1191,7 @@
1191
"targets": [
1192
{
1193
"editorMode": "code",
1194
- "expr": "histogram_quantile(0.50, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__range])))",
1194
+ "expr": "histogram_quantile(0.50, sum by(le) (rate(vllm:inter_token_latency_seconds_bucket[$__range])))",
1195
"legendFormat": "__auto",
1196
"range": true,
1197
"refId": "A"
examples/online_serving/dashboards/perses/performance_statistics.yaml
+10
-10
@@ -309,9 +309,9 @@ spec:
309
kind: PrometheusDatasource
310
name: accelerators-thanos-querier-datasource
311
query: >
312
- sum by (model_name) (rate(vllm:time_per_output_token_seconds_sum{model_name=~"$Deployment_id"}[$__interval]))
312
+ sum by (model_name) (rate(vllm:inter_token_latency_seconds_sum{model_name=~"$Deployment_id"}[$__interval]))
313
/
314
- sum by (model_name) (rate(vllm:time_per_output_token_seconds_count{model_name=~"$Deployment_id"}[$__interval]))
314
+ sum by (model_name) (rate(vllm:inter_token_latency_seconds_count{model_name=~"$Deployment_id"}[$__interval]))
315
seriesNameFormat: '{{model_name}}'
316
- kind: TimeSeriesQuery
317
spec:
@@ -325,7 +325,7 @@ spec:
325
histogram_quantile(
326
0.50,
327
sum by (le, model_name) (
328
- rate(vllm:time_per_output_token_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
328
+ rate(vllm:inter_token_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
329
)
330
)
331
seriesNameFormat: '{{model_name}} p50'
@@ -341,7 +341,7 @@ spec:
341
histogram_quantile(
342
0.90,
343
sum by (le, model_name) (
344
- rate(vllm:time_per_output_token_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
344
+ rate(vllm:inter_token_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
345
)
346
)
347
seriesNameFormat: '{{model_name}} p90'
@@ -357,7 +357,7 @@ spec:
357
histogram_quantile(
358
0.99,
359
sum by (le, model_name) (
360
- rate(vllm:time_per_output_token_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
360
+ rate(vllm:inter_token_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
361
)
362
)
363
seriesNameFormat: '{{model_name}} p99'
@@ -381,9 +381,9 @@ spec:
381
kind: PrometheusDatasource
382
name: accelerators-thanos-querier-datasource
383
query: >
384
- (sum by (model_name) (increase(vllm:time_per_output_token_seconds_sum{model_name=~"$Deployment_id"}[$__range])))
384
+ (sum by (model_name) (increase(vllm:inter_token_latency_seconds_sum{model_name=~"$Deployment_id"}[$__range])))
385
/
386
- (sum by (model_name) (increase(vllm:time_per_output_token_seconds_count{model_name=~"$Deployment_id"}[$__range])))
386
+ (sum by (model_name) (increase(vllm:inter_token_latency_seconds_count{model_name=~"$Deployment_id"}[$__range])))
387
388
"13":
389
kind: Panel
@@ -407,7 +407,7 @@ spec:
407
histogram_quantile(
408
0.50,
409
sum by (le, model_name) (
410
- rate(vllm:time_per_output_token_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
410
+ rate(vllm:inter_token_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
411
)
412
)
413
@@ -433,7 +433,7 @@ spec:
433
histogram_quantile(
434
0.90,
435
sum by (le, model_name) (
436
- rate(vllm:time_per_output_token_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
436
+ rate(vllm:inter_token_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
437
)
438
)
439
@@ -459,7 +459,7 @@ spec:
459
histogram_quantile(
460
0.99,
461
sum by (le, model_name) (
462
- rate(vllm:time_per_output_token_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
462
+ rate(vllm:inter_token_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
463
)
464
)
465
tests/entrypoints/instrumentator/test_metrics.py
+1
-7
@@ -77,7 +77,7 @@ def _get_expected_values(num_requests: int, prompt_ids: list[int], max_tokens: i
77
# {metric_family: [(suffix, expected_value)]}
78
return {
79
"vllm:time_to_first_token_seconds": [("_count", num_requests)],
80
- "vllm:time_per_output_token_seconds": [
80
+ "vllm:inter_token_latency_seconds": [
81
("_count", num_requests * (max_tokens - 1))
82
],
83
"vllm:e2e_request_latency_seconds": [("_count", num_requests)],
@@ -203,9 +203,6 @@ EXPECTED_METRICS_V1 = [
203
"vllm:request_params_max_tokens_sum",
204
"vllm:request_params_max_tokens_bucket",
205
"vllm:request_params_max_tokens_count",
206
- "vllm:time_per_output_token_seconds_sum",
207
- "vllm:time_per_output_token_seconds_bucket",
208
- "vllm:time_per_output_token_seconds_count",
206
"vllm:time_to_first_token_seconds_sum",
207
"vllm:time_to_first_token_seconds_bucket",
208
"vllm:time_to_first_token_seconds_count",
@@ -238,9 +235,6 @@ HIDDEN_DEPRECATED_METRICS: list[str] = [
235
"vllm:gpu_cache_usage_perc",
236
"vllm:gpu_prefix_cache_queries",
237
"vllm:gpu_prefix_cache_hits",
241
- "vllm:time_per_output_token_seconds_sum",
242
- "vllm:time_per_output_token_seconds_bucket",
243
- "vllm:time_per_output_token_seconds_count",
238
]
239
240
vllm/v1/metrics/loggers.py
-39
@@ -715,43 +715,6 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
715
histogram_time_to_first_token, engine_indexes, model_name
716
)
717
718
- # Deprecated in 0.11 - Renamed as vllm:inter_token_latency_seconds
719
- # With 0.12.x you can enable with --show-hidden-metrics-for-version=0.11
720
- # TODO: remove in 0.13.0
721
- if self.show_hidden_metrics:
722
- histogram_time_per_output_token = self._histogram_cls(
723
- name="vllm:time_per_output_token_seconds",
724
- documentation=(
725
- "Histogram of time per output token in seconds."
726
- "DEPRECATED: Use vllm:inter_token_latency_seconds instead."
727
- ),
728
- buckets=[
729
- 0.01,
730
- 0.025,
731
- 0.05,
732
- 0.075,
733
- 0.1,
734
- 0.15,
735
- 0.2,
736
- 0.3,
737
- 0.4,
738
- 0.5,
739
- 0.75,
740
- 1.0,
741
- 2.5,
742
- 5.0,
743
- 7.5,
744
- 10.0,
745
- 20.0,
746
- 40.0,
747
- 80.0,
748
- ],
749
- labelnames=labelnames,
750
- )
751
- self.histogram_time_per_output_token = make_per_engine(
752
- histogram_time_per_output_token, engine_indexes, model_name
753
- )
754
-
718
histogram_inter_token_latency = self._histogram_cls(
719
name="vllm:inter_token_latency_seconds",
720
documentation="Histogram of inter-token latency in seconds.",
@@ -1124,8 +1087,6 @@ class PrometheusStatLogger(AggregateStatLoggerBase):
1087
self.histogram_time_to_first_token[engine_idx].observe(ttft)
1088
for itl in iteration_stats.inter_token_latencies_iter:
1089
self.histogram_inter_token_latency[engine_idx].observe(itl)
1127
- if self.show_hidden_metrics:
1128
- self.histogram_time_per_output_token[engine_idx].observe(itl)
1090
1091
for finished_request in iteration_stats.finished_requests:
1092
self.counter_request_success[finished_request.finish_reason][