@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 874f551b36
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Metrics] add more metrics (#4464)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: Robert Shaw <rshaw@neuralmagic.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

harrywu committed Nov 12, 2024 at 00:17 UTC 874f551b3626321f6bf9a902b8fd9fc1fa7c7f2e
4 files changed +437 -50
examples/production_monitoring/grafana.json
+343 -41
@@ -1,33 +1,4 @@
1 {
2 - "__inputs": [
3 - ],
4 - "__elements": {},
5 - "__requires": [
6 - {
7 - "type": "grafana",
8 - "id": "grafana",
9 - "name": "Grafana",
10 - "version": "10.4.2"
11 - },
12 - {
13 - "type": "panel",
14 - "id": "heatmap",
15 - "name": "Heatmap",
16 - "version": ""
17 - },
18 - {
19 - "type": "datasource",
20 - "id": "prometheus",
21 - "name": "Prometheus",
22 - "version": "1.0.0"
23 - },
24 - {
25 - "type": "panel",
26 - "id": "timeseries",
27 - "name": "Time series",
28 - "version": ""
29 - }
30 - ],
2 "annotations": {
3 "list": [
4 {
@@ -54,7 +25,7 @@
25 "editable": true,
26 "fiscalYearStartMonth": 0,
27 "graphTooltip": 0,
57 - "id": null,
28 + "id": 1,
29 "links": [],
30 "liveNow": false,
31 "panels": [
@@ -76,6 +47,7 @@
47 "axisLabel": "",
48 "axisPlacement": "auto",
49 "barAlignment": 0,
50 + "barWidthFactor": 0.6,
51 "drawStyle": "line",
52 "fillOpacity": 0,
53 "gradientMode": "none",
@@ -241,6 +213,7 @@
213 "axisLabel": "",
214 "axisPlacement": "auto",
215 "barAlignment": 0,
216 + "barWidthFactor": 0.6,
217 "drawStyle": "line",
218 "fillOpacity": 0,
219 "gradientMode": "none",
@@ -358,6 +331,7 @@
331 "axisLabel": "",
332 "axisPlacement": "auto",
333 "barAlignment": 0,
334 + "barWidthFactor": 0.6,
335 "drawStyle": "line",
336 "fillOpacity": 0,
337 "gradientMode": "none",
@@ -523,6 +497,7 @@
497 "axisLabel": "",
498 "axisPlacement": "auto",
499 "barAlignment": 0,
500 + "barWidthFactor": 0.6,
501 "drawStyle": "line",
502 "fillOpacity": 0,
503 "gradientMode": "none",
@@ -658,6 +633,7 @@
633 "axisLabel": "",
634 "axisPlacement": "auto",
635 "barAlignment": 0,
636 + "barWidthFactor": 0.6,
637 "drawStyle": "line",
638 "fillOpacity": 0,
639 "gradientMode": "none",
@@ -823,6 +799,7 @@
799 "axisLabel": "",
800 "axisPlacement": "auto",
801 "barAlignment": 0,
802 + "barWidthFactor": 0.6,
803 "drawStyle": "line",
804 "fillOpacity": 0,
805 "gradientMode": "none",
@@ -984,7 +961,7 @@
961 "unit": "none"
962 }
963 },
987 - "pluginVersion": "10.4.2",
964 + "pluginVersion": "11.2.0",
965 "targets": [
966 {
967 "datasource": {
@@ -1076,7 +1053,7 @@
1053 "unit": "none"
1054 }
1055 },
1079 - "pluginVersion": "10.4.2",
1056 + "pluginVersion": "11.2.0",
1057 "targets": [
1058 {
1059 "datasource": {
@@ -1117,6 +1094,7 @@
1094 "axisLabel": "",
1095 "axisPlacement": "auto",
1096 "barAlignment": 0,
1097 + "barWidthFactor": 0.6,
1098 "drawStyle": "line",
1099 "fillOpacity": 0,
1100 "gradientMode": "none",
@@ -1147,8 +1125,7 @@
1125 "mode": "absolute",
1126 "steps": [
1127 {
1150 - "color": "green",
1151 - "value": null
1128 + "color": "green"
1129 },
1130 {
1131 "color": "red",
@@ -1199,6 +1176,319 @@
1176 ],
1177 "title": "Finish Reason",
1178 "type": "timeseries"
1179 + },
1180 + {
1181 + "datasource": {
1182 + "default": false,
1183 + "type": "prometheus",
1184 + "uid": "${DS_PROMETHEUS}"
1185 + },
1186 + "fieldConfig": {
1187 + "defaults": {
1188 + "color": {
1189 + "mode": "palette-classic"
1190 + },
1191 + "custom": {
1192 + "axisBorderShow": false,
1193 + "axisCenteredZero": false,
1194 + "axisColorMode": "text",
1195 + "axisLabel": "seconds",
1196 + "axisPlacement": "auto",
1197 + "barAlignment": 0,
1198 + "barWidthFactor": 0.6,
1199 + "drawStyle": "line",
1200 + "fillOpacity": 0,
1201 + "gradientMode": "none",
1202 + "hideFrom": {
1203 + "legend": false,
1204 + "tooltip": false,
1205 + "viz": false
1206 + },
1207 + "insertNulls": false,
1208 + "lineInterpolation": "linear",
1209 + "lineWidth": 1,
1210 + "pointSize": 5,
1211 + "scaleDistribution": {
1212 + "type": "linear"
1213 + },
1214 + "showPoints": "auto",
1215 + "spanNulls": false,
1216 + "stacking": {
1217 + "group": "A",
1218 + "mode": "none"
1219 + },
1220 + "thresholdsStyle": {
1221 + "mode": "off"
1222 + }
1223 + },
1224 + "mappings": [],
1225 + "thresholds": {
1226 + "mode": "absolute",
1227 + "steps": [
1228 + {
1229 + "color": "green"
1230 + },
1231 + {
1232 + "color": "red",
1233 + "value": 80
1234 + }
1235 + ]
1236 + }
1237 + },
1238 + "overrides": []
1239 + },
1240 + "gridPos": {
1241 + "h": 8,
1242 + "w": 12,
1243 + "x": 12,
1244 + "y": 32
1245 + },
1246 + "id": 14,
1247 + "options": {
1248 + "legend": {
1249 + "calcs": [],
1250 + "displayMode": "list",
1251 + "placement": "bottom",
1252 + "showLegend": true
1253 + },
1254 + "tooltip": {
1255 + "mode": "single",
1256 + "sort": "none"
1257 + }
1258 + },
1259 + "targets": [
1260 + {
1261 + "datasource": {
1262 + "type": "prometheus",
1263 + "uid": "edx8memhpd9tsa"
1264 + },
1265 + "disableTextWrap": false,
1266 + "editorMode": "code",
1267 + "expr": "rate(vllm:request_queue_time_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])",
1268 + "fullMetaSearch": false,
1269 + "includeNullMetadata": true,
1270 + "instant": false,
1271 + "legendFormat": "__auto",
1272 + "range": true,
1273 + "refId": "A",
1274 + "useBackend": false
1275 + }
1276 + ],
1277 + "title": "Queue Time",
1278 + "type": "timeseries"
1279 + },
1280 + {
1281 + "datasource": {
1282 + "default": false,
1283 + "type": "prometheus",
1284 + "uid": "${DS_PROMETHEUS}"
1285 + },
1286 + "fieldConfig": {
1287 + "defaults": {
1288 + "color": {
1289 + "mode": "palette-classic"
1290 + },
1291 + "custom": {
1292 + "axisBorderShow": false,
1293 + "axisCenteredZero": false,
1294 + "axisColorMode": "text",
1295 + "axisLabel": "",
1296 + "axisPlacement": "auto",
1297 + "barAlignment": 0,
1298 + "barWidthFactor": 0.6,
1299 + "drawStyle": "line",
1300 + "fillOpacity": 0,
1301 + "gradientMode": "none",
1302 + "hideFrom": {
1303 + "legend": false,
1304 + "tooltip": false,
1305 + "viz": false
1306 + },
1307 + "insertNulls": false,
1308 + "lineInterpolation": "linear",
1309 + "lineWidth": 1,
1310 + "pointSize": 5,
1311 + "scaleDistribution": {
1312 + "type": "linear"
1313 + },
1314 + "showPoints": "auto",
1315 + "spanNulls": false,
1316 + "stacking": {
1317 + "group": "A",
1318 + "mode": "none"
1319 + },
1320 + "thresholdsStyle": {
1321 + "mode": "off"
1322 + }
1323 + },
1324 + "mappings": [],
1325 + "thresholds": {
1326 + "mode": "absolute",
1327 + "steps": [
1328 + {
1329 + "color": "green"
1330 + },
1331 + {
1332 + "color": "red",
1333 + "value": 80
1334 + }
1335 + ]
1336 + }
1337 + },
1338 + "overrides": []
1339 + },
1340 + "gridPos": {
1341 + "h": 8,
1342 + "w": 12,
1343 + "x": 0,
1344 + "y": 40
1345 + },
1346 + "id": 15,
1347 + "options": {
1348 + "legend": {
1349 + "calcs": [],
1350 + "displayMode": "list",
1351 + "placement": "bottom",
1352 + "showLegend": true
1353 + },
1354 + "tooltip": {
1355 + "mode": "single",
1356 + "sort": "none"
1357 + }
1358 + },
1359 + "targets": [
1360 + {
1361 + "datasource": {
1362 + "type": "prometheus",
1363 + "uid": "edx8memhpd9tsa"
1364 + },
1365 + "disableTextWrap": false,
1366 + "editorMode": "code",
1367 + "expr": "rate(vllm:request_prefill_time_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])",
1368 + "fullMetaSearch": false,
1369 + "includeNullMetadata": true,
1370 + "instant": false,
1371 + "legendFormat": "Prefill",
1372 + "range": true,
1373 + "refId": "A",
1374 + "useBackend": false
1375 + },
1376 + {
1377 + "datasource": {
1378 + "type": "prometheus",
1379 + "uid": "${DS_PROMETHEUS}"
1380 + },
1381 + "editorMode": "code",
1382 + "expr": "rate(vllm:request_decode_time_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])",
1383 + "hide": false,
1384 + "instant": false,
1385 + "legendFormat": "Decode",
1386 + "range": true,
1387 + "refId": "B"
1388 + }
1389 + ],
1390 + "title": "Requests Prefill and Decode Time",
1391 + "type": "timeseries"
1392 + },
1393 + {
1394 + "datasource": {
1395 + "default": false,
1396 + "type": "prometheus",
1397 + "uid": "${DS_PROMETHEUS}"
1398 + },
1399 + "fieldConfig": {
1400 + "defaults": {
1401 + "color": {
1402 + "mode": "palette-classic"
1403 + },
1404 + "custom": {
1405 + "axisBorderShow": false,
1406 + "axisCenteredZero": false,
1407 + "axisColorMode": "text",
1408 + "axisLabel": "",
1409 + "axisPlacement": "auto",
1410 + "barAlignment": 0,
1411 + "barWidthFactor": 0.6,
1412 + "drawStyle": "line",
1413 + "fillOpacity": 0,
1414 + "gradientMode": "none",
1415 + "hideFrom": {
1416 + "legend": false,
1417 + "tooltip": false,
1418 + "viz": false
1419 + },
1420 + "insertNulls": false,
1421 + "lineInterpolation": "linear",
1422 + "lineWidth": 1,
1423 + "pointSize": 5,
1424 + "scaleDistribution": {
1425 + "type": "linear"
1426 + },
1427 + "showPoints": "auto",
1428 + "spanNulls": false,
1429 + "stacking": {
1430 + "group": "A",
1431 + "mode": "none"
1432 + },
1433 + "thresholdsStyle": {
1434 + "mode": "off"
1435 + }
1436 + },
1437 + "mappings": [],
1438 + "thresholds": {
1439 + "mode": "absolute",
1440 + "steps": [
1441 + {
1442 + "color": "green"
1443 + },
1444 + {
1445 + "color": "red",
1446 + "value": 80
1447 + }
1448 + ]
1449 + }
1450 + },
1451 + "overrides": []
1452 + },
1453 + "gridPos": {
1454 + "h": 8,
1455 + "w": 12,
1456 + "x": 12,
1457 + "y": 40
1458 + },
1459 + "id": 16,
1460 + "options": {
1461 + "legend": {
1462 + "calcs": [],
1463 + "displayMode": "list",
1464 + "placement": "bottom",
1465 + "showLegend": true
1466 + },
1467 + "tooltip": {
1468 + "mode": "single",
1469 + "sort": "none"
1470 + }
1471 + },
1472 + "targets": [
1473 + {
1474 + "datasource": {
1475 + "type": "prometheus",
1476 + "uid": "edx8memhpd9tsa"
1477 + },
1478 + "disableTextWrap": false,
1479 + "editorMode": "code",
1480 + "expr": "rate(vllm:request_max_num_generation_tokens_sum{model_name=\"$model_name\"}[$__rate_interval])",
1481 + "fullMetaSearch": false,
1482 + "includeNullMetadata": true,
1483 + "instant": false,
1484 + "legendFormat": "Tokens",
1485 + "range": true,
1486 + "refId": "A",
1487 + "useBackend": false
1488 + }
1489 + ],
1490 + "title": "Max Generation Token in Sequence Group",
1491 + "type": "timeseries"
1492 }
1493 ],
1494 "refresh": "",
@@ -1207,21 +1497,34 @@
1497 "templating": {
1498 "list": [
1499 {
1210 - "type": "datasource",
1211 - "name": "DS_PROMETHEUS",
1212 - "label": "datasource",
1213 - "current": {},
1500 + "current": {
1501 + "selected": false,
1502 + "text": "prometheus",
1503 + "value": "edx8memhpd9tsa"
1504 + },
1505 "hide": 0,
1506 "includeAll": false,
1507 + "label": "datasource",
1508 "multi": false,
1509 + "name": "DS_PROMETHEUS",
1510 "options": [],
1511 "query": "prometheus",
1512 "queryValue": "",
1513 "refresh": 1,
1514 "regex": "",
1222 - "skipUrlSync": false
1515 + "skipUrlSync": false,
1516 + "type": "datasource"
1517 },
1518 {
1519 + "current": {
1520 + "selected": false,
1521 + "text": "/share/datasets/public_models/Meta-Llama-3-8B-Instruct",
1522 + "value": "/share/datasets/public_models/Meta-Llama-3-8B-Instruct"
1523 + },
1524 + "datasource": {
1525 + "type": "prometheus",
1526 + "uid": "edx8memhpd9tsa"
1527 + },
1528 "definition": "label_values(model_name)",
1529 "hide": 0,
1530 "includeAll": false,
@@ -1249,7 +1552,6 @@
1552 "timezone": "",
1553 "title": "vLLM",
1554 "uid": "b281712d-8bff-41ef-9f3f-71ad43c05e9b",
1252 - "version": 1,
1555 + "version": 8,
1556 "weekStart": ""
1557 }
1255 -
vllm/engine/llm_engine.py
+30 -1
@@ -1672,6 +1672,7 @@ class LLMEngine:
1672 # Iteration stats
1673 num_prompt_tokens_iter = 0
1674 num_generation_tokens_iter = 0
1675 + num_tokens_iter = 0
1676 time_to_first_tokens_iter: List[float] = []
1677 time_per_output_tokens_iter: List[float] = []
1678 num_preemption_iter = (0 if scheduler_outputs is None else
@@ -1680,6 +1681,10 @@ class LLMEngine:
1681 # Request stats
1682 # Latency
1683 time_e2e_requests: List[float] = []
1684 + time_queue_requests: List[float] = []
1685 + time_inference_requests: List[float] = []
1686 + time_prefill_requests: List[float] = []
1687 + time_decode_requests: List[float] = []
1688 time_in_queue_requests: List[float] = []
1689 model_forward_time_requests: List[float] = []
1690 model_execute_time_requests: List[float] = []
@@ -1687,6 +1692,7 @@ class LLMEngine:
1692 num_prompt_tokens_requests: List[int] = []
1693 num_generation_tokens_requests: List[int] = []
1694 n_requests: List[int] = []
1695 + max_num_generation_tokens_requests: List[int] = []
1696 max_tokens_requests: List[int] = []
1697 finished_reason_requests: List[str] = []
1698
@@ -1777,6 +1783,18 @@ class LLMEngine:
1783 # Latency timings
1784 time_e2e_requests.append(now -
1785 seq_group.metrics.arrival_time)
1786 + if (seq_group.metrics.first_scheduled_time is not None and
1787 + seq_group.metrics.first_token_time is not None):
1788 + time_queue_requests.append(
1789 + seq_group.metrics.first_scheduled_time -
1790 + seq_group.metrics.arrival_time)
1791 + time_prefill_requests.append(
1792 + seq_group.metrics.first_token_time -
1793 + seq_group.metrics.first_scheduled_time)
1794 + time_decode_requests.append(
1795 + now - seq_group.metrics.first_token_time)
1796 + time_inference_requests.append(
1797 + now - seq_group.metrics.first_scheduled_time)
1798 if seq_group.metrics.time_in_queue is not None:
1799 time_in_queue_requests.append(
1800 seq_group.metrics.time_in_queue)
@@ -1793,6 +1811,9 @@ class LLMEngine:
1811 seq.get_output_len()
1812 for seq in seq_group.get_finished_seqs()
1813 ])
1814 + max_num_generation_tokens_requests.append(
1815 + max(seq.get_output_len()
1816 + for seq in seq_group.get_seqs()))
1817 if seq_group.sampling_params is not None:
1818 n_requests.append(seq_group.sampling_params.n)
1819 max_tokens_requests.append(
@@ -1811,7 +1832,8 @@ class LLMEngine:
1832 num_generation_tokens_iter = (
1833 actual_num_batched_tokens - num_prompt_tokens_iter +
1834 num_generation_tokens_from_prefill_groups)
1814 -
1835 + num_tokens_iter = (num_generation_tokens_iter +
1836 + num_prompt_tokens_iter)
1837 # Spec decode, if enabled, emits specialized metrics from the worker in
1838 # sampler output.
1839 if model_output and (model_output[0].spec_decode_worker_metrics
@@ -1837,6 +1859,7 @@ class LLMEngine:
1859 # Iteration stats
1860 num_prompt_tokens_iter=num_prompt_tokens_iter,
1861 num_generation_tokens_iter=num_generation_tokens_iter,
1862 + num_tokens_iter=num_tokens_iter,
1863 time_to_first_tokens_iter=time_to_first_tokens_iter,
1864 time_per_output_tokens_iter=time_per_output_tokens_iter,
1865 spec_decode_metrics=spec_decode_metrics,
@@ -1845,12 +1868,18 @@ class LLMEngine:
1868 # Request stats
1869 # Latency
1870 time_e2e_requests=time_e2e_requests,
1871 + time_queue_requests=time_queue_requests,
1872 + time_inference_requests=time_inference_requests,
1873 + time_prefill_requests=time_prefill_requests,
1874 + time_decode_requests=time_decode_requests,
1875 time_in_queue_requests=time_in_queue_requests,
1876 model_forward_time_requests=model_forward_time_requests,
1877 model_execute_time_requests=model_execute_time_requests,
1878 # Metadata
1879 num_prompt_tokens_requests=num_prompt_tokens_requests,
1880 num_generation_tokens_requests=num_generation_tokens_requests,
1881 + max_num_generation_tokens_requests=
1882 + max_num_generation_tokens_requests,
1883 n_requests=n_requests,
1884 max_tokens_requests=max_tokens_requests,
1885 finished_reason_requests=finished_reason_requests,
vllm/engine/metrics.py
+58 -8
@@ -111,6 +111,15 @@ class Metrics:
111 name="vllm:generation_tokens_total",
112 documentation="Number of generation tokens processed.",
113 labelnames=labelnames)
114 + self.counter_tokens = self._counter_cls(
115 + name="vllm:tokens_total",
116 + documentation="Number of prefill plus generation tokens processed.",
117 + labelnames=labelnames)
118 + self.histogram_iteration_tokens = self._histogram_cls(
119 + name="vllm:iteration_tokens_total",
120 + documentation="Histogram of number of tokens per engine_step.",
121 + labelnames=labelnames,
122 + buckets=[1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096])
123 self.histogram_time_to_first_token = self._histogram_cls(
124 name="vllm:time_to_first_token_seconds",
125 documentation="Histogram of time to first token in seconds.",
@@ -130,23 +139,45 @@ class Metrics:
139
140 # Request stats
141 # Latency
142 + request_latency_buckets = [
143 + 0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
144 + 40.0, 50.0, 60.0
145 + ]
146 self.histogram_e2e_time_request = self._histogram_cls(
147 name="vllm:e2e_request_latency_seconds",
148 documentation="Histogram of end to end request latency in seconds.",
149 labelnames=labelnames,
137 - buckets=[
138 - 0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
139 - 40.0, 50.0, 60.0
140 - ])
150 + buckets=request_latency_buckets)
151 + self.histogram_queue_time_request = self._histogram_cls(
152 + name="vllm:request_queue_time_seconds",
153 + documentation=
154 + "Histogram of time spent in WAITING phase for request.",
155 + labelnames=labelnames,
156 + buckets=request_latency_buckets)
157 + self.histogram_inference_time_request = self._histogram_cls(
158 + name="vllm:request_inference_time_seconds",
159 + documentation=
160 + "Histogram of time spent in RUNNING phase for request.",
161 + labelnames=labelnames,
162 + buckets=request_latency_buckets)
163 + self.histogram_prefill_time_request = self._histogram_cls(
164 + name="vllm:request_prefill_time_seconds",
165 + documentation=
166 + "Histogram of time spent in PREFILL phase for request.",
167 + labelnames=labelnames,
168 + buckets=request_latency_buckets)
169 + self.histogram_decode_time_request = self._histogram_cls(
170 + name="vllm:request_decode_time_seconds",
171 + documentation=
172 + "Histogram of time spent in DECODE phase for request.",
173 + labelnames=labelnames,
174 + buckets=request_latency_buckets)
175 self.histogram_time_in_queue_request = self._histogram_cls(
176 name="vllm:time_in_queue_requests",
177 documentation=
178 "Histogram of time the request spent in the queue in seconds.",
179 labelnames=labelnames,
146 - buckets=[
147 - 0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
148 - 40.0, 50.0, 60.0
149 - ])
180 + buckets=request_latency_buckets)
181 self.histogram_model_forward_time_request = self._histogram_cls(
182 name="vllm:model_forward_time_milliseconds",
183 documentation=
@@ -173,6 +204,12 @@ class Metrics:
204 labelnames=labelnames,
205 buckets=build_1_2_5_buckets(max_model_len),
206 )
207 + self.histogram_max_num_generation_tokens_request = self._histogram_cls(
208 + name="vllm:request_max_num_generation_tokens",
209 + documentation=
210 + "Histogram of maximum number of requested generation tokens.",
211 + labelnames=labelnames,
212 + buckets=build_1_2_5_buckets(max_model_len))
213 self.histogram_n_request = self._histogram_cls(
214 name="vllm:request_params_n",
215 documentation="Histogram of the n request parameter.",
@@ -526,6 +563,8 @@ class PrometheusStatLogger(StatLoggerBase):
563 stats.num_prompt_tokens_iter)
564 self._log_counter(self.metrics.counter_generation_tokens,
565 stats.num_generation_tokens_iter)
566 + self._log_histogram(self.metrics.histogram_iteration_tokens,
567 + [stats.num_tokens_iter])
568 self._log_histogram(self.metrics.histogram_time_to_first_token,
569 stats.time_to_first_tokens_iter)
570 self._log_histogram(self.metrics.histogram_time_per_output_token,
@@ -535,6 +574,14 @@ class PrometheusStatLogger(StatLoggerBase):
574 # Latency
575 self._log_histogram(self.metrics.histogram_e2e_time_request,
576 stats.time_e2e_requests)
577 + self._log_histogram(self.metrics.histogram_queue_time_request,
578 + stats.time_queue_requests)
579 + self._log_histogram(self.metrics.histogram_inference_time_request,
580 + stats.time_inference_requests)
581 + self._log_histogram(self.metrics.histogram_decode_time_request,
582 + stats.time_prefill_requests)
583 + self._log_histogram(self.metrics.histogram_prefill_time_request,
584 + stats.time_decode_requests)
585 self._log_histogram(self.metrics.histogram_time_in_queue_request,
586 stats.time_in_queue_requests)
587 self._log_histogram(self.metrics.histogram_model_forward_time_request,
@@ -553,6 +600,9 @@ class PrometheusStatLogger(StatLoggerBase):
600 self.metrics.histogram_num_generation_tokens_request,
601 stats.num_generation_tokens_requests)
602 self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
603 + self._log_histogram(
604 + self.metrics.histogram_max_num_generation_tokens_request,
605 + stats.max_num_generation_tokens_requests)
606 self._log_histogram(self.metrics.histogram_max_tokens_request,
607 stats.max_tokens_requests)
608
vllm/engine/metrics_types.py
+6
@@ -39,6 +39,7 @@ class Stats:
39 # Iteration stats (should have _iter suffix)
40 num_prompt_tokens_iter: int
41 num_generation_tokens_iter: int
42 + num_tokens_iter: int
43 time_to_first_tokens_iter: List[float]
44 time_per_output_tokens_iter: List[float]
45 num_preemption_iter: int
@@ -46,6 +47,10 @@ class Stats:
47 # Request stats (should have _requests suffix)
48 # Latency
49 time_e2e_requests: List[float]
50 + time_queue_requests: List[float]
51 + time_inference_requests: List[float]
52 + time_prefill_requests: List[float]
53 + time_decode_requests: List[float]
54 time_in_queue_requests: List[float]
55 model_forward_time_requests: List[float]
56 model_execute_time_requests: List[float]
@@ -53,6 +58,7 @@ class Stats:
58 num_prompt_tokens_requests: List[int]
59 num_generation_tokens_requests: List[int]
60 n_requests: List[int]
61 + max_num_generation_tokens_requests: List[int]
62 max_tokens_requests: List[int]
63 finished_reason_requests: List[str]
64 waiting_lora_adapters: List[str]