A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Metrics] Add bucket for `request_latency`, `time_to_first_token` and `time_per_output_token` (#15202)
Signed-off-by: Kay Yan <kay.yan@daocloud.io>
Kay Yan committed
Apr 7, 2025 at 11:34 UTC
86fc2321ff1e1bd2ba15193053bf4515a19bb481
2 files changed
+8
-6
vllm/engine/metrics.py
+4
-3
@@ -156,7 +156,8 @@ class Metrics:
156
labelnames=labelnames,
157
buckets=[
158
0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
159
- 0.75, 1.0, 2.5, 5.0, 7.5, 10.0
159
+ 0.75, 1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0, 80.0, 160.0, 640.0,
160
+ 2560.0
161
])
162
self.histogram_time_per_output_token = self._histogram_cls(
163
name="vllm:time_per_output_token_seconds",
@@ -164,14 +165,14 @@ class Metrics:
165
labelnames=labelnames,
166
buckets=[
167
0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
167
- 1.0, 2.5
168
+ 1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0, 80.0
169
])
170
171
# Request stats
172
# Latency
173
request_latency_buckets = [
174
0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
174
- 40.0, 50.0, 60.0
175
+ 40.0, 50.0, 60.0, 120.0, 240.0, 480.0, 960.0, 1920.0, 7680.0
176
]
177
self.histogram_e2e_time_request = self._histogram_cls(
178
name="vllm:e2e_request_latency_seconds",
vllm/v1/metrics/loggers.py
+4
-3
@@ -239,7 +239,8 @@ class PrometheusStatLogger(StatLoggerBase):
239
documentation="Histogram of time to first token in seconds.",
240
buckets=[
241
0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
242
- 0.75, 1.0, 2.5, 5.0, 7.5, 10.0
242
+ 0.75, 1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0, 80.0, 160.0,
243
+ 640.0, 2560.0
244
],
245
labelnames=labelnames).labels(*labelvalues)
246
@@ -249,13 +250,13 @@ class PrometheusStatLogger(StatLoggerBase):
250
documentation="Histogram of time per output token in seconds.",
251
buckets=[
252
0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5,
252
- 0.75, 1.0, 2.5
253
+ 0.75, 1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0, 80.0
254
],
255
labelnames=labelnames).labels(*labelvalues)
256
257
request_latency_buckets = [
258
0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
258
- 40.0, 50.0, 60.0
259
+ 40.0, 50.0, 60.0, 120.0, 240.0, 480.0, 960.0, 1920.0, 7680.0
260
]
261
self.histogram_e2e_time_request = \
262
prometheus_client.Histogram(