A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Metrics] add more metrics (#4464)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Robert Shaw <rshaw@neuralmagic.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
harrywu committed
Nov 12, 2024 at 00:17 UTC
874f551b3626321f6bf9a902b8fd9fc1fa7c7f2e
4 files changed
+437
-50
examples/production_monitoring/grafana.json
+343
-41
@@ -1,33 +1,4 @@
1
{
2
- "__inputs": [
3
- ],
4
- "__elements": {},
5
- "__requires": [
6
- {
7
- "type": "grafana",
8
- "id": "grafana",
9
- "name": "Grafana",
10
- "version": "10.4.2"
11
- },
12
- {
13
- "type": "panel",
14
- "id": "heatmap",
15
- "name": "Heatmap",
16
- "version": ""
17
- },
18
- {
19
- "type": "datasource",
20
- "id": "prometheus",
21
- "name": "Prometheus",
22
- "version": "1.0.0"
23
- },
24
- {
25
- "type": "panel",
26
- "id": "timeseries",
27
- "name": "Time series",
28
- "version": ""
29
- }
30
- ],
2
"annotations": {
3
"list": [
4
{
@@ -54,7 +25,7 @@
25
"editable": true,
26
"fiscalYearStartMonth": 0,
27
"graphTooltip": 0,
57
- "id": null,
28
+ "id": 1,
29
"links": [],
30
"liveNow": false,
31
"panels": [
@@ -76,6 +47,7 @@
47
"axisLabel": "",
48
"axisPlacement": "auto",
49
"barAlignment": 0,
50
+ "barWidthFactor": 0.6,
51
"drawStyle": "line",
52
"fillOpacity": 0,
53
"gradientMode": "none",
@@ -241,6 +213,7 @@
213
"axisLabel": "",
214
"axisPlacement": "auto",
215
"barAlignment": 0,
216
+ "barWidthFactor": 0.6,
217
"drawStyle": "line",
218
"fillOpacity": 0,
219
"gradientMode": "none",
@@ -358,6 +331,7 @@
331
"axisLabel": "",
332
"axisPlacement": "auto",
333
"barAlignment": 0,
334
+ "barWidthFactor": 0.6,
335
"drawStyle": "line",
336
"fillOpacity": 0,
337
"gradientMode": "none",
@@ -523,6 +497,7 @@
497
"axisLabel": "",
498
"axisPlacement": "auto",
499
"barAlignment": 0,
500
+ "barWidthFactor": 0.6,
501
"drawStyle": "line",
502
"fillOpacity": 0,
503
"gradientMode": "none",
@@ -658,6 +633,7 @@
633
"axisLabel": "",
634
"axisPlacement": "auto",
635
"barAlignment": 0,
636
+ "barWidthFactor": 0.6,
637
"drawStyle": "line",
638
"fillOpacity": 0,
639
"gradientMode": "none",
@@ -823,6 +799,7 @@
799
"axisLabel": "",
800
"axisPlacement": "auto",
801
"barAlignment": 0,
802
+ "barWidthFactor": 0.6,
803
"drawStyle": "line",
804
"fillOpacity": 0,
805
"gradientMode": "none",
@@ -984,7 +961,7 @@
961
"unit": "none"
962
}
963
},
987
- "pluginVersion": "10.4.2",
964
+ "pluginVersion": "11.2.0",
965
"targets": [
966
{
967
"datasource": {
@@ -1076,7 +1053,7 @@
1053
"unit": "none"
1054
}
1055
},
1079
- "pluginVersion": "10.4.2",
1056
+ "pluginVersion": "11.2.0",
1057
"targets": [
1058
{
1059
"datasource": {
@@ -1117,6 +1094,7 @@
1094
"axisLabel": "",
1095
"axisPlacement": "auto",
1096
"barAlignment": 0,
1097
+ "barWidthFactor": 0.6,
1098
"drawStyle": "line",
1099
"fillOpacity": 0,
1100
"gradientMode": "none",
@@ -1147,8 +1125,7 @@
1125
"mode": "absolute",
1126
"steps": [
1127
{
1150
- "color": "green",
1151
- "value": null
1128
+ "color": "green"
1129
},
1130
{
1131
"color": "red",
@@ -1199,6 +1176,319 @@
1176
],
1177
"title": "Finish Reason",
1178
"type": "timeseries"
1179
+ },
1180
+ {
1181
+ "datasource": {
1182
+ "default": false,
1183
+ "type": "prometheus",
1184
+ "uid": "${DS_PROMETHEUS}"
1185
+ },
1186
+ "fieldConfig": {
1187
+ "defaults": {
1188
+ "color": {
1189
+ "mode": "palette-classic"
1190
+ },
1191
+ "custom": {
1192
+ "axisBorderShow": false,
1193
+ "axisCenteredZero": false,
1194
+ "axisColorMode": "text",
1195
+ "axisLabel": "seconds",
1196
+ "axisPlacement": "auto",
1197
+ "barAlignment": 0,
1198
+ "barWidthFactor": 0.6,
1199
+ "drawStyle": "line",
1200
+ "fillOpacity": 0,
1201
+ "gradientMode": "none",
1202
+ "hideFrom": {
1203
+ "legend": false,
1204
+ "tooltip": false,
1205
+ "viz": false
1206
+ },
1207
+ "insertNulls": false,
1208
+ "lineInterpolation": "linear",
1209
+ "lineWidth": 1,
1210
+ "pointSize": 5,
1211
+ "scaleDistribution": {
1212
+ "type": "linear"
1213
+ },
1214
+ "showPoints": "auto",
1215
+ "spanNulls": false,
1216
+ "stacking": {
1217
+ "group": "A",
1218
+ "mode": "none"
1219
+ },
1220
+ "thresholdsStyle": {
1221
+ "mode": "off"
1222
+ }
1223
+ },
1224
+ "mappings": [],
1225
+ "thresholds": {
1226
+ "mode": "absolute",
1227
+ "steps": [
1228
+ {
1229
+ "color": "green"
1230
+ },
1231
+ {
1232
+ "color": "red",
1233
+ "value": 80
1234
+ }
1235
+ ]
1236
+ }
1237
+ },
1238
+ "overrides": []
1239
+ },
1240
+ "gridPos": {
1241
+ "h": 8,
1242
+ "w": 12,
1243
+ "x": 12,
1244
+ "y": 32
1245
+ },
1246
+ "id": 14,
1247
+ "options": {
1248
+ "legend": {
1249
+ "calcs": [],
1250
+ "displayMode": "list",
1251
+ "placement": "bottom",
1252
+ "showLegend": true
1253
+ },
1254
+ "tooltip": {
1255
+ "mode": "single",
1256
+ "sort": "none"
1257
+ }
1258
+ },
1259
+ "targets": [
1260
+ {
1261
+ "datasource": {
1262
+ "type": "prometheus",
1263
+ "uid": "edx8memhpd9tsa"
1264
+ },
1265
+ "disableTextWrap": false,
1266
+ "editorMode": "code",
1267
+ "expr": "rate(vllm:request_queue_time_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])",
1268
+ "fullMetaSearch": false,
1269
+ "includeNullMetadata": true,
1270
+ "instant": false,
1271
+ "legendFormat": "__auto",
1272
+ "range": true,
1273
+ "refId": "A",
1274
+ "useBackend": false
1275
+ }
1276
+ ],
1277
+ "title": "Queue Time",
1278
+ "type": "timeseries"
1279
+ },
1280
+ {
1281
+ "datasource": {
1282
+ "default": false,
1283
+ "type": "prometheus",
1284
+ "uid": "${DS_PROMETHEUS}"
1285
+ },
1286
+ "fieldConfig": {
1287
+ "defaults": {
1288
+ "color": {
1289
+ "mode": "palette-classic"
1290
+ },
1291
+ "custom": {
1292
+ "axisBorderShow": false,
1293
+ "axisCenteredZero": false,
1294
+ "axisColorMode": "text",
1295
+ "axisLabel": "",
1296
+ "axisPlacement": "auto",
1297
+ "barAlignment": 0,
1298
+ "barWidthFactor": 0.6,
1299
+ "drawStyle": "line",
1300
+ "fillOpacity": 0,
1301
+ "gradientMode": "none",
1302
+ "hideFrom": {
1303
+ "legend": false,
1304
+ "tooltip": false,
1305
+ "viz": false
1306
+ },
1307
+ "insertNulls": false,
1308
+ "lineInterpolation": "linear",
1309
+ "lineWidth": 1,
1310
+ "pointSize": 5,
1311
+ "scaleDistribution": {
1312
+ "type": "linear"
1313
+ },
1314
+ "showPoints": "auto",
1315
+ "spanNulls": false,
1316
+ "stacking": {
1317
+ "group": "A",
1318
+ "mode": "none"
1319
+ },
1320
+ "thresholdsStyle": {
1321
+ "mode": "off"
1322
+ }
1323
+ },
1324
+ "mappings": [],
1325
+ "thresholds": {
1326
+ "mode": "absolute",
1327
+ "steps": [
1328
+ {
1329
+ "color": "green"
1330
+ },
1331
+ {
1332
+ "color": "red",
1333
+ "value": 80
1334
+ }
1335
+ ]
1336
+ }
1337
+ },
1338
+ "overrides": []
1339
+ },
1340
+ "gridPos": {
1341
+ "h": 8,
1342
+ "w": 12,
1343
+ "x": 0,
1344
+ "y": 40
1345
+ },
1346
+ "id": 15,
1347
+ "options": {
1348
+ "legend": {
1349
+ "calcs": [],
1350
+ "displayMode": "list",
1351
+ "placement": "bottom",
1352
+ "showLegend": true
1353
+ },
1354
+ "tooltip": {
1355
+ "mode": "single",
1356
+ "sort": "none"
1357
+ }
1358
+ },
1359
+ "targets": [
1360
+ {
1361
+ "datasource": {
1362
+ "type": "prometheus",
1363
+ "uid": "edx8memhpd9tsa"
1364
+ },
1365
+ "disableTextWrap": false,
1366
+ "editorMode": "code",
1367
+ "expr": "rate(vllm:request_prefill_time_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])",
1368
+ "fullMetaSearch": false,
1369
+ "includeNullMetadata": true,
1370
+ "instant": false,
1371
+ "legendFormat": "Prefill",
1372
+ "range": true,
1373
+ "refId": "A",
1374
+ "useBackend": false
1375
+ },
1376
+ {
1377
+ "datasource": {
1378
+ "type": "prometheus",
1379
+ "uid": "${DS_PROMETHEUS}"
1380
+ },
1381
+ "editorMode": "code",
1382
+ "expr": "rate(vllm:request_decode_time_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])",
1383
+ "hide": false,
1384
+ "instant": false,
1385
+ "legendFormat": "Decode",
1386
+ "range": true,
1387
+ "refId": "B"
1388
+ }
1389
+ ],
1390
+ "title": "Requests Prefill and Decode Time",
1391
+ "type": "timeseries"
1392
+ },
1393
+ {
1394
+ "datasource": {
1395
+ "default": false,
1396
+ "type": "prometheus",
1397
+ "uid": "${DS_PROMETHEUS}"
1398
+ },
1399
+ "fieldConfig": {
1400
+ "defaults": {
1401
+ "color": {
1402
+ "mode": "palette-classic"
1403
+ },
1404
+ "custom": {
1405
+ "axisBorderShow": false,
1406
+ "axisCenteredZero": false,
1407
+ "axisColorMode": "text",
1408
+ "axisLabel": "",
1409
+ "axisPlacement": "auto",
1410
+ "barAlignment": 0,
1411
+ "barWidthFactor": 0.6,
1412
+ "drawStyle": "line",
1413
+ "fillOpacity": 0,
1414
+ "gradientMode": "none",
1415
+ "hideFrom": {
1416
+ "legend": false,
1417
+ "tooltip": false,
1418
+ "viz": false
1419
+ },
1420
+ "insertNulls": false,
1421
+ "lineInterpolation": "linear",
1422
+ "lineWidth": 1,
1423
+ "pointSize": 5,
1424
+ "scaleDistribution": {
1425
+ "type": "linear"
1426
+ },
1427
+ "showPoints": "auto",
1428
+ "spanNulls": false,
1429
+ "stacking": {
1430
+ "group": "A",
1431
+ "mode": "none"
1432
+ },
1433
+ "thresholdsStyle": {
1434
+ "mode": "off"
1435
+ }
1436
+ },
1437
+ "mappings": [],
1438
+ "thresholds": {
1439
+ "mode": "absolute",
1440
+ "steps": [
1441
+ {
1442
+ "color": "green"
1443
+ },
1444
+ {
1445
+ "color": "red",
1446
+ "value": 80
1447
+ }
1448
+ ]
1449
+ }
1450
+ },
1451
+ "overrides": []
1452
+ },
1453
+ "gridPos": {
1454
+ "h": 8,
1455
+ "w": 12,
1456
+ "x": 12,
1457
+ "y": 40
1458
+ },
1459
+ "id": 16,
1460
+ "options": {
1461
+ "legend": {
1462
+ "calcs": [],
1463
+ "displayMode": "list",
1464
+ "placement": "bottom",
1465
+ "showLegend": true
1466
+ },
1467
+ "tooltip": {
1468
+ "mode": "single",
1469
+ "sort": "none"
1470
+ }
1471
+ },
1472
+ "targets": [
1473
+ {
1474
+ "datasource": {
1475
+ "type": "prometheus",
1476
+ "uid": "edx8memhpd9tsa"
1477
+ },
1478
+ "disableTextWrap": false,
1479
+ "editorMode": "code",
1480
+ "expr": "rate(vllm:request_max_num_generation_tokens_sum{model_name=\"$model_name\"}[$__rate_interval])",
1481
+ "fullMetaSearch": false,
1482
+ "includeNullMetadata": true,
1483
+ "instant": false,
1484
+ "legendFormat": "Tokens",
1485
+ "range": true,
1486
+ "refId": "A",
1487
+ "useBackend": false
1488
+ }
1489
+ ],
1490
+ "title": "Max Generation Token in Sequence Group",
1491
+ "type": "timeseries"
1492
}
1493
],
1494
"refresh": "",
@@ -1207,21 +1497,34 @@
1497
"templating": {
1498
"list": [
1499
{
1210
- "type": "datasource",
1211
- "name": "DS_PROMETHEUS",
1212
- "label": "datasource",
1213
- "current": {},
1500
+ "current": {
1501
+ "selected": false,
1502
+ "text": "prometheus",
1503
+ "value": "edx8memhpd9tsa"
1504
+ },
1505
"hide": 0,
1506
"includeAll": false,
1507
+ "label": "datasource",
1508
"multi": false,
1509
+ "name": "DS_PROMETHEUS",
1510
"options": [],
1511
"query": "prometheus",
1512
"queryValue": "",
1513
"refresh": 1,
1514
"regex": "",
1222
- "skipUrlSync": false
1515
+ "skipUrlSync": false,
1516
+ "type": "datasource"
1517
},
1518
{
1519
+ "current": {
1520
+ "selected": false,
1521
+ "text": "/share/datasets/public_models/Meta-Llama-3-8B-Instruct",
1522
+ "value": "/share/datasets/public_models/Meta-Llama-3-8B-Instruct"
1523
+ },
1524
+ "datasource": {
1525
+ "type": "prometheus",
1526
+ "uid": "edx8memhpd9tsa"
1527
+ },
1528
"definition": "label_values(model_name)",
1529
"hide": 0,
1530
"includeAll": false,
@@ -1249,7 +1552,6 @@
1552
"timezone": "",
1553
"title": "vLLM",
1554
"uid": "b281712d-8bff-41ef-9f3f-71ad43c05e9b",
1252
- "version": 1,
1555
+ "version": 8,
1556
"weekStart": ""
1557
}
1255
-
vllm/engine/llm_engine.py
+30
-1
@@ -1672,6 +1672,7 @@ class LLMEngine:
1672
# Iteration stats
1673
num_prompt_tokens_iter = 0
1674
num_generation_tokens_iter = 0
1675
+ num_tokens_iter = 0
1676
time_to_first_tokens_iter: List[float] = []
1677
time_per_output_tokens_iter: List[float] = []
1678
num_preemption_iter = (0 if scheduler_outputs is None else
@@ -1680,6 +1681,10 @@ class LLMEngine:
1681
# Request stats
1682
# Latency
1683
time_e2e_requests: List[float] = []
1684
+ time_queue_requests: List[float] = []
1685
+ time_inference_requests: List[float] = []
1686
+ time_prefill_requests: List[float] = []
1687
+ time_decode_requests: List[float] = []
1688
time_in_queue_requests: List[float] = []
1689
model_forward_time_requests: List[float] = []
1690
model_execute_time_requests: List[float] = []
@@ -1687,6 +1692,7 @@ class LLMEngine:
1692
num_prompt_tokens_requests: List[int] = []
1693
num_generation_tokens_requests: List[int] = []
1694
n_requests: List[int] = []
1695
+ max_num_generation_tokens_requests: List[int] = []
1696
max_tokens_requests: List[int] = []
1697
finished_reason_requests: List[str] = []
1698
@@ -1777,6 +1783,18 @@ class LLMEngine:
1783
# Latency timings
1784
time_e2e_requests.append(now -
1785
seq_group.metrics.arrival_time)
1786
+ if (seq_group.metrics.first_scheduled_time is not None and
1787
+ seq_group.metrics.first_token_time is not None):
1788
+ time_queue_requests.append(
1789
+ seq_group.metrics.first_scheduled_time -
1790
+ seq_group.metrics.arrival_time)
1791
+ time_prefill_requests.append(
1792
+ seq_group.metrics.first_token_time -
1793
+ seq_group.metrics.first_scheduled_time)
1794
+ time_decode_requests.append(
1795
+ now - seq_group.metrics.first_token_time)
1796
+ time_inference_requests.append(
1797
+ now - seq_group.metrics.first_scheduled_time)
1798
if seq_group.metrics.time_in_queue is not None:
1799
time_in_queue_requests.append(
1800
seq_group.metrics.time_in_queue)
@@ -1793,6 +1811,9 @@ class LLMEngine:
1811
seq.get_output_len()
1812
for seq in seq_group.get_finished_seqs()
1813
])
1814
+ max_num_generation_tokens_requests.append(
1815
+ max(seq.get_output_len()
1816
+ for seq in seq_group.get_seqs()))
1817
if seq_group.sampling_params is not None:
1818
n_requests.append(seq_group.sampling_params.n)
1819
max_tokens_requests.append(
@@ -1811,7 +1832,8 @@ class LLMEngine:
1832
num_generation_tokens_iter = (
1833
actual_num_batched_tokens - num_prompt_tokens_iter +
1834
num_generation_tokens_from_prefill_groups)
1814
-
1835
+ num_tokens_iter = (num_generation_tokens_iter +
1836
+ num_prompt_tokens_iter)
1837
# Spec decode, if enabled, emits specialized metrics from the worker in
1838
# sampler output.
1839
if model_output and (model_output[0].spec_decode_worker_metrics
@@ -1837,6 +1859,7 @@ class LLMEngine:
1859
# Iteration stats
1860
num_prompt_tokens_iter=num_prompt_tokens_iter,
1861
num_generation_tokens_iter=num_generation_tokens_iter,
1862
+ num_tokens_iter=num_tokens_iter,
1863
time_to_first_tokens_iter=time_to_first_tokens_iter,
1864
time_per_output_tokens_iter=time_per_output_tokens_iter,
1865
spec_decode_metrics=spec_decode_metrics,
@@ -1845,12 +1868,18 @@ class LLMEngine:
1868
# Request stats
1869
# Latency
1870
time_e2e_requests=time_e2e_requests,
1871
+ time_queue_requests=time_queue_requests,
1872
+ time_inference_requests=time_inference_requests,
1873
+ time_prefill_requests=time_prefill_requests,
1874
+ time_decode_requests=time_decode_requests,
1875
time_in_queue_requests=time_in_queue_requests,
1876
model_forward_time_requests=model_forward_time_requests,
1877
model_execute_time_requests=model_execute_time_requests,
1878
# Metadata
1879
num_prompt_tokens_requests=num_prompt_tokens_requests,
1880
num_generation_tokens_requests=num_generation_tokens_requests,
1881
+ max_num_generation_tokens_requests=
1882
+ max_num_generation_tokens_requests,
1883
n_requests=n_requests,
1884
max_tokens_requests=max_tokens_requests,
1885
finished_reason_requests=finished_reason_requests,
vllm/engine/metrics.py
+58
-8
@@ -111,6 +111,15 @@ class Metrics:
111
name="vllm:generation_tokens_total",
112
documentation="Number of generation tokens processed.",
113
labelnames=labelnames)
114
+ self.counter_tokens = self._counter_cls(
115
+ name="vllm:tokens_total",
116
+ documentation="Number of prefill plus generation tokens processed.",
117
+ labelnames=labelnames)
118
+ self.histogram_iteration_tokens = self._histogram_cls(
119
+ name="vllm:iteration_tokens_total",
120
+ documentation="Histogram of number of tokens per engine_step.",
121
+ labelnames=labelnames,
122
+ buckets=[1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096])
123
self.histogram_time_to_first_token = self._histogram_cls(
124
name="vllm:time_to_first_token_seconds",
125
documentation="Histogram of time to first token in seconds.",
@@ -130,23 +139,45 @@ class Metrics:
139
140
# Request stats
141
# Latency
142
+ request_latency_buckets = [
143
+ 0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
144
+ 40.0, 50.0, 60.0
145
+ ]
146
self.histogram_e2e_time_request = self._histogram_cls(
147
name="vllm:e2e_request_latency_seconds",
148
documentation="Histogram of end to end request latency in seconds.",
149
labelnames=labelnames,
137
- buckets=[
138
- 0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
139
- 40.0, 50.0, 60.0
140
- ])
150
+ buckets=request_latency_buckets)
151
+ self.histogram_queue_time_request = self._histogram_cls(
152
+ name="vllm:request_queue_time_seconds",
153
+ documentation=
154
+ "Histogram of time spent in WAITING phase for request.",
155
+ labelnames=labelnames,
156
+ buckets=request_latency_buckets)
157
+ self.histogram_inference_time_request = self._histogram_cls(
158
+ name="vllm:request_inference_time_seconds",
159
+ documentation=
160
+ "Histogram of time spent in RUNNING phase for request.",
161
+ labelnames=labelnames,
162
+ buckets=request_latency_buckets)
163
+ self.histogram_prefill_time_request = self._histogram_cls(
164
+ name="vllm:request_prefill_time_seconds",
165
+ documentation=
166
+ "Histogram of time spent in PREFILL phase for request.",
167
+ labelnames=labelnames,
168
+ buckets=request_latency_buckets)
169
+ self.histogram_decode_time_request = self._histogram_cls(
170
+ name="vllm:request_decode_time_seconds",
171
+ documentation=
172
+ "Histogram of time spent in DECODE phase for request.",
173
+ labelnames=labelnames,
174
+ buckets=request_latency_buckets)
175
self.histogram_time_in_queue_request = self._histogram_cls(
176
name="vllm:time_in_queue_requests",
177
documentation=
178
"Histogram of time the request spent in the queue in seconds.",
179
labelnames=labelnames,
146
- buckets=[
147
- 0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
148
- 40.0, 50.0, 60.0
149
- ])
180
+ buckets=request_latency_buckets)
181
self.histogram_model_forward_time_request = self._histogram_cls(
182
name="vllm:model_forward_time_milliseconds",
183
documentation=
@@ -173,6 +204,12 @@ class Metrics:
204
labelnames=labelnames,
205
buckets=build_1_2_5_buckets(max_model_len),
206
)
207
+ self.histogram_max_num_generation_tokens_request = self._histogram_cls(
208
+ name="vllm:request_max_num_generation_tokens",
209
+ documentation=
210
+ "Histogram of maximum number of requested generation tokens.",
211
+ labelnames=labelnames,
212
+ buckets=build_1_2_5_buckets(max_model_len))
213
self.histogram_n_request = self._histogram_cls(
214
name="vllm:request_params_n",
215
documentation="Histogram of the n request parameter.",
@@ -526,6 +563,8 @@ class PrometheusStatLogger(StatLoggerBase):
563
stats.num_prompt_tokens_iter)
564
self._log_counter(self.metrics.counter_generation_tokens,
565
stats.num_generation_tokens_iter)
566
+ self._log_histogram(self.metrics.histogram_iteration_tokens,
567
+ [stats.num_tokens_iter])
568
self._log_histogram(self.metrics.histogram_time_to_first_token,
569
stats.time_to_first_tokens_iter)
570
self._log_histogram(self.metrics.histogram_time_per_output_token,
@@ -535,6 +574,14 @@ class PrometheusStatLogger(StatLoggerBase):
574
# Latency
575
self._log_histogram(self.metrics.histogram_e2e_time_request,
576
stats.time_e2e_requests)
577
+ self._log_histogram(self.metrics.histogram_queue_time_request,
578
+ stats.time_queue_requests)
579
+ self._log_histogram(self.metrics.histogram_inference_time_request,
580
+ stats.time_inference_requests)
581
+ self._log_histogram(self.metrics.histogram_decode_time_request,
582
+ stats.time_prefill_requests)
583
+ self._log_histogram(self.metrics.histogram_prefill_time_request,
584
+ stats.time_decode_requests)
585
self._log_histogram(self.metrics.histogram_time_in_queue_request,
586
stats.time_in_queue_requests)
587
self._log_histogram(self.metrics.histogram_model_forward_time_request,
@@ -553,6 +600,9 @@ class PrometheusStatLogger(StatLoggerBase):
600
self.metrics.histogram_num_generation_tokens_request,
601
stats.num_generation_tokens_requests)
602
self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
603
+ self._log_histogram(
604
+ self.metrics.histogram_max_num_generation_tokens_request,
605
+ stats.max_num_generation_tokens_requests)
606
self._log_histogram(self.metrics.histogram_max_tokens_request,
607
stats.max_tokens_requests)
608
vllm/engine/metrics_types.py
+6
@@ -39,6 +39,7 @@ class Stats:
39
# Iteration stats (should have _iter suffix)
40
num_prompt_tokens_iter: int
41
num_generation_tokens_iter: int
42
+ num_tokens_iter: int
43
time_to_first_tokens_iter: List[float]
44
time_per_output_tokens_iter: List[float]
45
num_preemption_iter: int
@@ -46,6 +47,10 @@ class Stats:
47
# Request stats (should have _requests suffix)
48
# Latency
49
time_e2e_requests: List[float]
50
+ time_queue_requests: List[float]
51
+ time_inference_requests: List[float]
52
+ time_prefill_requests: List[float]
53
+ time_decode_requests: List[float]
54
time_in_queue_requests: List[float]
55
model_forward_time_requests: List[float]
56
model_execute_time_requests: List[float]
@@ -53,6 +58,7 @@ class Stats:
58
num_prompt_tokens_requests: List[int]
59
num_generation_tokens_requests: List[int]
60
n_requests: List[int]
61
+ max_num_generation_tokens_requests: List[int]
62
max_tokens_requests: List[int]
63
finished_reason_requests: List[str]
64
waiting_lora_adapters: List[str]