A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Refactor Prometheus and Add Request Level Metrics (#2316)
Robert Shaw committed
Jan 31, 2024 at 14:58 UTC
93b38bea5dd03e1b140ca997dfaadef86f8f1855
7 files changed
+1234
-102
examples/production_monitoring/README.md
new
+54
@@ -0,0 +1,54 @@
1
+# vLLM + Prometheus/Grafana
2
+
3
+This is a simple example that shows you how to connect vLLM metric logging to the Prometheus/Grafana stack. For this example, we launch Prometheus and Grafana via Docker. You can checkout other methods through [Prometheus](https://prometheus.io/) and [Grafana](https://grafana.com/) websites.
4
+
5
+Install:
6
+- [`docker`](https://docs.docker.com/engine/install/)
7
+- [`docker compose`](https://docs.docker.com/compose/install/linux/#install-using-the-repository)
8
+
9
+### Launch
10
+
11
+Prometheus metric logging is enabled by default in the OpenAI-compatible server. Launch via the entrypoint:
12
+```bash
13
+python3 -m vllm.entrypoints.openai.api_server \
14
+ --model mistralai/Mistral-7B-v0.1 \
15
+ --max-model-len 2048 \
16
+ --disable-log-requests
17
+```
18
+
19
+Launch Prometheus and Grafana servers with `docker compose`:
20
+```bash
21
+docker compose up
22
+```
23
+
24
+Submit some sample requests to the server:
25
+```bash
26
+wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
27
+
28
+python3 ../../benchmarks/benchmark_serving.py \
29
+ --model mistralai/Mistral-7B-v0.1 \
30
+ --tokenizer mistralai/Mistral-7B-v0.1 \
31
+ --endpoint /v1/completions \
32
+ --dataset ShareGPT_V3_unfiltered_cleaned_split.json \
33
+ --request-rate 3.0
34
+```
35
+
36
+Navigating to [`http://localhost:8000/metrics`](http://localhost:8000/metrics) will show the raw Prometheus metrics being exposed by vLLM.
37
+
38
+### Grafana Dashboard
39
+
40
+Navigate to [`http://localhost:3000`](http://localhost:3000). Log in with the default username (`admin`) and password (`admin`).
41
+
42
+#### Add Prometheus Data Source
43
+
44
+Navigate to [`http://localhost:3000/connections/datasources/new`](http://localhost:3000/connections/datasources/new) and select Prometheus.
45
+
46
+On Prometheus configuration page, we need to add the `Prometheus Server URL` in `Connection`. For this setup, Grafana and Prometheus are running in separate containers, but Docker creates DNS name for each containers. You can just use `http://prometheus:9090`.
47
+
48
+Click `Save & Test`. You should get a green check saying "Successfully queried the Prometheus API.".
49
+
50
+#### Import Dashboard
51
+
52
+Navigate to [`http://localhost:3000/dashboard/import`](http://localhost:3000/dashboard/import), upload `grafana.json`, and select the `prometheus` datasource. You should see a screen that looks like the following:
53
+
54
+
examples/production_monitoring/docker-compose.yaml
new
+19
@@ -0,0 +1,19 @@
1
+# docker-compose.yaml
2
+version: "3"
3
+
4
+services:
5
+ prometheus:
6
+ image: prom/prometheus:latest
7
+ extra_hosts:
8
+ - "host.docker.internal:host-gateway" # allow a direct connection from container to the local machine
9
+ ports:
10
+ - "9090:9090" # the default port used by Prometheus
11
+ volumes:
12
+ - ${PWD}/prometheus.yaml:/etc/prometheus/prometheus.yml # mount Prometheus config file
13
+
14
+ grafana:
15
+ image: grafana/grafana:latest
16
+ depends_on:
17
+ - prometheus
18
+ ports:
19
+ - "3000:3000" # the default port used by Grafana
examples/production_monitoring/grafana.json
new
+931
@@ -0,0 +1,931 @@
1
+{
2
+ "__inputs": [
3
+ {
4
+ "name": "DS_PROMETHEUS",
5
+ "label": "prometheus",
6
+ "description": "",
7
+ "type": "datasource",
8
+ "pluginId": "prometheus",
9
+ "pluginName": "Prometheus"
10
+ }
11
+ ],
12
+ "__elements": {},
13
+ "__requires": [
14
+ {
15
+ "type": "grafana",
16
+ "id": "grafana",
17
+ "name": "Grafana",
18
+ "version": "10.2.3"
19
+ },
20
+ {
21
+ "type": "datasource",
22
+ "id": "prometheus",
23
+ "name": "Prometheus",
24
+ "version": "1.0.0"
25
+ },
26
+ {
27
+ "type": "panel",
28
+ "id": "timeseries",
29
+ "name": "Time series",
30
+ "version": ""
31
+ }
32
+ ],
33
+ "annotations": {
34
+ "list": [
35
+ {
36
+ "builtIn": 1,
37
+ "datasource": {
38
+ "type": "grafana",
39
+ "uid": "-- Grafana --"
40
+ },
41
+ "enable": true,
42
+ "hide": true,
43
+ "iconColor": "rgba(0, 211, 255, 1)",
44
+ "name": "Annotations & Alerts",
45
+ "type": "dashboard"
46
+ }
47
+ ]
48
+ },
49
+ "description": "Monitoring vLLM Inference Server",
50
+ "editable": true,
51
+ "fiscalYearStartMonth": 0,
52
+ "graphTooltip": 0,
53
+ "id": null,
54
+ "links": [],
55
+ "liveNow": false,
56
+ "panels": [
57
+ {
58
+ "datasource": {
59
+ "type": "prometheus",
60
+ "uid": "${DS_PROMETHEUS}"
61
+ },
62
+ "description": "End to end request latency measured in seconds.",
63
+ "fieldConfig": {
64
+ "defaults": {
65
+ "color": {
66
+ "mode": "palette-classic"
67
+ },
68
+ "custom": {
69
+ "axisBorderShow": false,
70
+ "axisCenteredZero": false,
71
+ "axisColorMode": "text",
72
+ "axisLabel": "",
73
+ "axisPlacement": "auto",
74
+ "barAlignment": 0,
75
+ "drawStyle": "line",
76
+ "fillOpacity": 0,
77
+ "gradientMode": "none",
78
+ "hideFrom": {
79
+ "legend": false,
80
+ "tooltip": false,
81
+ "viz": false
82
+ },
83
+ "insertNulls": false,
84
+ "lineInterpolation": "linear",
85
+ "lineWidth": 1,
86
+ "pointSize": 5,
87
+ "scaleDistribution": {
88
+ "type": "linear"
89
+ },
90
+ "showPoints": "auto",
91
+ "spanNulls": false,
92
+ "stacking": {
93
+ "group": "A",
94
+ "mode": "none"
95
+ },
96
+ "thresholdsStyle": {
97
+ "mode": "off"
98
+ }
99
+ },
100
+ "mappings": [],
101
+ "thresholds": {
102
+ "mode": "absolute",
103
+ "steps": [
104
+ {
105
+ "color": "green",
106
+ "value": null
107
+ },
108
+ {
109
+ "color": "red",
110
+ "value": 80
111
+ }
112
+ ]
113
+ },
114
+ "unit": "s"
115
+ },
116
+ "overrides": []
117
+ },
118
+ "gridPos": {
119
+ "h": 8,
120
+ "w": 12,
121
+ "x": 0,
122
+ "y": 0
123
+ },
124
+ "id": 9,
125
+ "options": {
126
+ "legend": {
127
+ "calcs": [],
128
+ "displayMode": "list",
129
+ "placement": "bottom",
130
+ "showLegend": true
131
+ },
132
+ "tooltip": {
133
+ "mode": "single",
134
+ "sort": "none"
135
+ }
136
+ },
137
+ "targets": [
138
+ {
139
+ "datasource": {
140
+ "type": "prometheus",
141
+ "uid": "${DS_PROMETHEUS}"
142
+ },
143
+ "disableTextWrap": false,
144
+ "editorMode": "builder",
145
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
146
+ "fullMetaSearch": false,
147
+ "includeNullMetadata": false,
148
+ "instant": false,
149
+ "legendFormat": "P99",
150
+ "range": true,
151
+ "refId": "A",
152
+ "useBackend": false
153
+ },
154
+ {
155
+ "datasource": {
156
+ "type": "prometheus",
157
+ "uid": "${DS_PROMETHEUS}"
158
+ },
159
+ "disableTextWrap": false,
160
+ "editorMode": "builder",
161
+ "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
162
+ "fullMetaSearch": false,
163
+ "hide": false,
164
+ "includeNullMetadata": false,
165
+ "instant": false,
166
+ "legendFormat": "P95",
167
+ "range": true,
168
+ "refId": "B",
169
+ "useBackend": false
170
+ },
171
+ {
172
+ "datasource": {
173
+ "type": "prometheus",
174
+ "uid": "${DS_PROMETHEUS}"
175
+ },
176
+ "disableTextWrap": false,
177
+ "editorMode": "builder",
178
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
179
+ "fullMetaSearch": false,
180
+ "hide": false,
181
+ "includeNullMetadata": false,
182
+ "instant": false,
183
+ "legendFormat": "P90",
184
+ "range": true,
185
+ "refId": "C",
186
+ "useBackend": false
187
+ },
188
+ {
189
+ "datasource": {
190
+ "type": "prometheus",
191
+ "uid": "${DS_PROMETHEUS}"
192
+ },
193
+ "disableTextWrap": false,
194
+ "editorMode": "builder",
195
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
196
+ "fullMetaSearch": false,
197
+ "hide": false,
198
+ "includeNullMetadata": false,
199
+ "instant": false,
200
+ "legendFormat": "P50",
201
+ "range": true,
202
+ "refId": "D",
203
+ "useBackend": false
204
+ },
205
+ {
206
+ "datasource": {
207
+ "type": "prometheus",
208
+ "uid": "${DS_PROMETHEUS}"
209
+ },
210
+ "editorMode": "code",
211
+ "expr": "rate(vllm:e2e_request_latency_seconds_sum[$__rate_interval])\n/\nrate(vllm:e2e_request_latency_seconds_count[$__rate_interval])",
212
+ "hide": false,
213
+ "instant": false,
214
+ "legendFormat": "Average",
215
+ "range": true,
216
+ "refId": "E"
217
+ }
218
+ ],
219
+ "title": "E2E Request Latency",
220
+ "type": "timeseries"
221
+ },
222
+ {
223
+ "datasource": {
224
+ "type": "prometheus",
225
+ "uid": "${DS_PROMETHEUS}"
226
+ },
227
+ "description": "Number of tokens processed per second",
228
+ "fieldConfig": {
229
+ "defaults": {
230
+ "color": {
231
+ "mode": "palette-classic"
232
+ },
233
+ "custom": {
234
+ "axisBorderShow": false,
235
+ "axisCenteredZero": false,
236
+ "axisColorMode": "text",
237
+ "axisLabel": "",
238
+ "axisPlacement": "auto",
239
+ "barAlignment": 0,
240
+ "drawStyle": "line",
241
+ "fillOpacity": 0,
242
+ "gradientMode": "none",
243
+ "hideFrom": {
244
+ "legend": false,
245
+ "tooltip": false,
246
+ "viz": false
247
+ },
248
+ "insertNulls": false,
249
+ "lineInterpolation": "linear",
250
+ "lineWidth": 1,
251
+ "pointSize": 5,
252
+ "scaleDistribution": {
253
+ "type": "linear"
254
+ },
255
+ "showPoints": "auto",
256
+ "spanNulls": false,
257
+ "stacking": {
258
+ "group": "A",
259
+ "mode": "none"
260
+ },
261
+ "thresholdsStyle": {
262
+ "mode": "off"
263
+ }
264
+ },
265
+ "mappings": [],
266
+ "thresholds": {
267
+ "mode": "absolute",
268
+ "steps": [
269
+ {
270
+ "color": "green",
271
+ "value": null
272
+ },
273
+ {
274
+ "color": "red",
275
+ "value": 80
276
+ }
277
+ ]
278
+ }
279
+ },
280
+ "overrides": []
281
+ },
282
+ "gridPos": {
283
+ "h": 8,
284
+ "w": 12,
285
+ "x": 12,
286
+ "y": 0
287
+ },
288
+ "id": 8,
289
+ "options": {
290
+ "legend": {
291
+ "calcs": [],
292
+ "displayMode": "list",
293
+ "placement": "bottom",
294
+ "showLegend": true
295
+ },
296
+ "tooltip": {
297
+ "mode": "single",
298
+ "sort": "none"
299
+ }
300
+ },
301
+ "targets": [
302
+ {
303
+ "datasource": {
304
+ "type": "prometheus",
305
+ "uid": "${DS_PROMETHEUS}"
306
+ },
307
+ "disableTextWrap": false,
308
+ "editorMode": "builder",
309
+ "expr": "rate(vllm:prompt_tokens_total[$__rate_interval])",
310
+ "fullMetaSearch": false,
311
+ "includeNullMetadata": false,
312
+ "instant": false,
313
+ "legendFormat": "Prompt Tokens/Sec",
314
+ "range": true,
315
+ "refId": "A",
316
+ "useBackend": false
317
+ },
318
+ {
319
+ "datasource": {
320
+ "type": "prometheus",
321
+ "uid": "${DS_PROMETHEUS}"
322
+ },
323
+ "disableTextWrap": false,
324
+ "editorMode": "builder",
325
+ "expr": "rate(vllm:generation_tokens_total[$__rate_interval])",
326
+ "fullMetaSearch": false,
327
+ "hide": false,
328
+ "includeNullMetadata": false,
329
+ "instant": false,
330
+ "legendFormat": "Generation Tokens/Sec",
331
+ "range": true,
332
+ "refId": "B",
333
+ "useBackend": false
334
+ }
335
+ ],
336
+ "title": "Token Throughput",
337
+ "type": "timeseries"
338
+ },
339
+ {
340
+ "datasource": {
341
+ "type": "prometheus",
342
+ "uid": "${DS_PROMETHEUS}"
343
+ },
344
+ "description": "Inter token latency in seconds.",
345
+ "fieldConfig": {
346
+ "defaults": {
347
+ "color": {
348
+ "mode": "palette-classic"
349
+ },
350
+ "custom": {
351
+ "axisBorderShow": false,
352
+ "axisCenteredZero": false,
353
+ "axisColorMode": "text",
354
+ "axisLabel": "",
355
+ "axisPlacement": "auto",
356
+ "barAlignment": 0,
357
+ "drawStyle": "line",
358
+ "fillOpacity": 0,
359
+ "gradientMode": "none",
360
+ "hideFrom": {
361
+ "legend": false,
362
+ "tooltip": false,
363
+ "viz": false
364
+ },
365
+ "insertNulls": false,
366
+ "lineInterpolation": "linear",
367
+ "lineWidth": 1,
368
+ "pointSize": 5,
369
+ "scaleDistribution": {
370
+ "type": "linear"
371
+ },
372
+ "showPoints": "auto",
373
+ "spanNulls": false,
374
+ "stacking": {
375
+ "group": "A",
376
+ "mode": "none"
377
+ },
378
+ "thresholdsStyle": {
379
+ "mode": "off"
380
+ }
381
+ },
382
+ "mappings": [],
383
+ "thresholds": {
384
+ "mode": "absolute",
385
+ "steps": [
386
+ {
387
+ "color": "green",
388
+ "value": null
389
+ },
390
+ {
391
+ "color": "red",
392
+ "value": 80
393
+ }
394
+ ]
395
+ },
396
+ "unit": "s"
397
+ },
398
+ "overrides": []
399
+ },
400
+ "gridPos": {
401
+ "h": 8,
402
+ "w": 12,
403
+ "x": 0,
404
+ "y": 8
405
+ },
406
+ "id": 10,
407
+ "options": {
408
+ "legend": {
409
+ "calcs": [],
410
+ "displayMode": "list",
411
+ "placement": "bottom",
412
+ "showLegend": true
413
+ },
414
+ "tooltip": {
415
+ "mode": "single",
416
+ "sort": "none"
417
+ }
418
+ },
419
+ "targets": [
420
+ {
421
+ "datasource": {
422
+ "type": "prometheus",
423
+ "uid": "${DS_PROMETHEUS}"
424
+ },
425
+ "disableTextWrap": false,
426
+ "editorMode": "builder",
427
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
428
+ "fullMetaSearch": false,
429
+ "includeNullMetadata": false,
430
+ "instant": false,
431
+ "legendFormat": "P99",
432
+ "range": true,
433
+ "refId": "A",
434
+ "useBackend": false
435
+ },
436
+ {
437
+ "datasource": {
438
+ "type": "prometheus",
439
+ "uid": "${DS_PROMETHEUS}"
440
+ },
441
+ "disableTextWrap": false,
442
+ "editorMode": "builder",
443
+ "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
444
+ "fullMetaSearch": false,
445
+ "hide": false,
446
+ "includeNullMetadata": false,
447
+ "instant": false,
448
+ "legendFormat": "P95",
449
+ "range": true,
450
+ "refId": "B",
451
+ "useBackend": false
452
+ },
453
+ {
454
+ "datasource": {
455
+ "type": "prometheus",
456
+ "uid": "${DS_PROMETHEUS}"
457
+ },
458
+ "disableTextWrap": false,
459
+ "editorMode": "builder",
460
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
461
+ "fullMetaSearch": false,
462
+ "hide": false,
463
+ "includeNullMetadata": false,
464
+ "instant": false,
465
+ "legendFormat": "P90",
466
+ "range": true,
467
+ "refId": "C",
468
+ "useBackend": false
469
+ },
470
+ {
471
+ "datasource": {
472
+ "type": "prometheus",
473
+ "uid": "${DS_PROMETHEUS}"
474
+ },
475
+ "disableTextWrap": false,
476
+ "editorMode": "builder",
477
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
478
+ "fullMetaSearch": false,
479
+ "hide": false,
480
+ "includeNullMetadata": false,
481
+ "instant": false,
482
+ "legendFormat": "P50",
483
+ "range": true,
484
+ "refId": "D",
485
+ "useBackend": false
486
+ },
487
+ {
488
+ "datasource": {
489
+ "type": "prometheus",
490
+ "uid": "${DS_PROMETHEUS}"
491
+ },
492
+ "editorMode": "code",
493
+ "expr": "rate(vllm:time_per_output_token_seconds_sum[$__rate_interval])\n/\nrate(vllm:time_per_output_token_seconds_count[$__rate_interval])",
494
+ "hide": false,
495
+ "instant": false,
496
+ "legendFormat": "Mean",
497
+ "range": true,
498
+ "refId": "E"
499
+ }
500
+ ],
501
+ "title": "Time Per Output Token Latency",
502
+ "type": "timeseries"
503
+ },
504
+ {
505
+ "datasource": {
506
+ "type": "prometheus",
507
+ "uid": "${DS_PROMETHEUS}"
508
+ },
509
+ "description": "Number of requests in RUNNING, WAITING, and SWAPPED state",
510
+ "fieldConfig": {
511
+ "defaults": {
512
+ "color": {
513
+ "mode": "palette-classic"
514
+ },
515
+ "custom": {
516
+ "axisBorderShow": false,
517
+ "axisCenteredZero": false,
518
+ "axisColorMode": "text",
519
+ "axisLabel": "",
520
+ "axisPlacement": "auto",
521
+ "barAlignment": 0,
522
+ "drawStyle": "line",
523
+ "fillOpacity": 0,
524
+ "gradientMode": "none",
525
+ "hideFrom": {
526
+ "legend": false,
527
+ "tooltip": false,
528
+ "viz": false
529
+ },
530
+ "insertNulls": false,
531
+ "lineInterpolation": "linear",
532
+ "lineWidth": 1,
533
+ "pointSize": 5,
534
+ "scaleDistribution": {
535
+ "type": "linear"
536
+ },
537
+ "showPoints": "auto",
538
+ "spanNulls": false,
539
+ "stacking": {
540
+ "group": "A",
541
+ "mode": "none"
542
+ },
543
+ "thresholdsStyle": {
544
+ "mode": "off"
545
+ }
546
+ },
547
+ "mappings": [],
548
+ "thresholds": {
549
+ "mode": "absolute",
550
+ "steps": [
551
+ {
552
+ "color": "green",
553
+ "value": null
554
+ },
555
+ {
556
+ "color": "red",
557
+ "value": 80
558
+ }
559
+ ]
560
+ },
561
+ "unit": "none"
562
+ },
563
+ "overrides": []
564
+ },
565
+ "gridPos": {
566
+ "h": 8,
567
+ "w": 12,
568
+ "x": 12,
569
+ "y": 8
570
+ },
571
+ "id": 3,
572
+ "options": {
573
+ "legend": {
574
+ "calcs": [],
575
+ "displayMode": "list",
576
+ "placement": "bottom",
577
+ "showLegend": true
578
+ },
579
+ "tooltip": {
580
+ "mode": "single",
581
+ "sort": "none"
582
+ }
583
+ },
584
+ "targets": [
585
+ {
586
+ "datasource": {
587
+ "type": "prometheus",
588
+ "uid": "${DS_PROMETHEUS}"
589
+ },
590
+ "disableTextWrap": false,
591
+ "editorMode": "builder",
592
+ "expr": "vllm:num_requests_running",
593
+ "fullMetaSearch": false,
594
+ "includeNullMetadata": true,
595
+ "instant": false,
596
+ "legendFormat": "Num Running",
597
+ "range": true,
598
+ "refId": "A",
599
+ "useBackend": false
600
+ },
601
+ {
602
+ "datasource": {
603
+ "type": "prometheus",
604
+ "uid": "${DS_PROMETHEUS}"
605
+ },
606
+ "disableTextWrap": false,
607
+ "editorMode": "builder",
608
+ "expr": "vllm:num_requests_swapped",
609
+ "fullMetaSearch": false,
610
+ "hide": false,
611
+ "includeNullMetadata": true,
612
+ "instant": false,
613
+ "legendFormat": "Num Swapped",
614
+ "range": true,
615
+ "refId": "B",
616
+ "useBackend": false
617
+ },
618
+ {
619
+ "datasource": {
620
+ "type": "prometheus",
621
+ "uid": "${DS_PROMETHEUS}"
622
+ },
623
+ "disableTextWrap": false,
624
+ "editorMode": "builder",
625
+ "expr": "vllm:num_requests_waiting",
626
+ "fullMetaSearch": false,
627
+ "hide": false,
628
+ "includeNullMetadata": true,
629
+ "instant": false,
630
+ "legendFormat": "Num Waiting",
631
+ "range": true,
632
+ "refId": "C",
633
+ "useBackend": false
634
+ }
635
+ ],
636
+ "title": "Scheduler State",
637
+ "type": "timeseries"
638
+ },
639
+ {
640
+ "datasource": {
641
+ "type": "prometheus",
642
+ "uid": "${DS_PROMETHEUS}"
643
+ },
644
+ "description": "P50, P90, P95, and P99 TTFT latency in seconds.",
645
+ "fieldConfig": {
646
+ "defaults": {
647
+ "color": {
648
+ "mode": "palette-classic"
649
+ },
650
+ "custom": {
651
+ "axisBorderShow": false,
652
+ "axisCenteredZero": false,
653
+ "axisColorMode": "text",
654
+ "axisLabel": "",
655
+ "axisPlacement": "auto",
656
+ "barAlignment": 0,
657
+ "drawStyle": "line",
658
+ "fillOpacity": 0,
659
+ "gradientMode": "none",
660
+ "hideFrom": {
661
+ "legend": false,
662
+ "tooltip": false,
663
+ "viz": false
664
+ },
665
+ "insertNulls": false,
666
+ "lineInterpolation": "linear",
667
+ "lineWidth": 1,
668
+ "pointSize": 5,
669
+ "scaleDistribution": {
670
+ "type": "linear"
671
+ },
672
+ "showPoints": "auto",
673
+ "spanNulls": false,
674
+ "stacking": {
675
+ "group": "A",
676
+ "mode": "none"
677
+ },
678
+ "thresholdsStyle": {
679
+ "mode": "off"
680
+ }
681
+ },
682
+ "mappings": [],
683
+ "thresholds": {
684
+ "mode": "absolute",
685
+ "steps": [
686
+ {
687
+ "color": "green",
688
+ "value": null
689
+ },
690
+ {
691
+ "color": "red",
692
+ "value": 80
693
+ }
694
+ ]
695
+ },
696
+ "unit": "s"
697
+ },
698
+ "overrides": []
699
+ },
700
+ "gridPos": {
701
+ "h": 8,
702
+ "w": 12,
703
+ "x": 0,
704
+ "y": 16
705
+ },
706
+ "id": 5,
707
+ "options": {
708
+ "legend": {
709
+ "calcs": [],
710
+ "displayMode": "list",
711
+ "placement": "bottom",
712
+ "showLegend": true
713
+ },
714
+ "tooltip": {
715
+ "mode": "single",
716
+ "sort": "none"
717
+ }
718
+ },
719
+ "targets": [
720
+ {
721
+ "datasource": {
722
+ "type": "prometheus",
723
+ "uid": "${DS_PROMETHEUS}"
724
+ },
725
+ "disableTextWrap": false,
726
+ "editorMode": "builder",
727
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
728
+ "fullMetaSearch": false,
729
+ "hide": false,
730
+ "includeNullMetadata": false,
731
+ "instant": false,
732
+ "legendFormat": "P99",
733
+ "range": true,
734
+ "refId": "A",
735
+ "useBackend": false
736
+ },
737
+ {
738
+ "datasource": {
739
+ "type": "prometheus",
740
+ "uid": "${DS_PROMETHEUS}"
741
+ },
742
+ "disableTextWrap": false,
743
+ "editorMode": "builder",
744
+ "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
745
+ "fullMetaSearch": false,
746
+ "includeNullMetadata": false,
747
+ "instant": false,
748
+ "legendFormat": "P95",
749
+ "range": true,
750
+ "refId": "B",
751
+ "useBackend": false
752
+ },
753
+ {
754
+ "datasource": {
755
+ "type": "prometheus",
756
+ "uid": "${DS_PROMETHEUS}"
757
+ },
758
+ "disableTextWrap": false,
759
+ "editorMode": "builder",
760
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
761
+ "fullMetaSearch": false,
762
+ "hide": false,
763
+ "includeNullMetadata": false,
764
+ "instant": false,
765
+ "legendFormat": "P90",
766
+ "range": true,
767
+ "refId": "C",
768
+ "useBackend": false
769
+ },
770
+ {
771
+ "datasource": {
772
+ "type": "prometheus",
773
+ "uid": "${DS_PROMETHEUS}"
774
+ },
775
+ "disableTextWrap": false,
776
+ "editorMode": "builder",
777
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
778
+ "fullMetaSearch": false,
779
+ "hide": false,
780
+ "includeNullMetadata": false,
781
+ "instant": false,
782
+ "legendFormat": "P50",
783
+ "range": true,
784
+ "refId": "D",
785
+ "useBackend": false
786
+ },
787
+ {
788
+ "datasource": {
789
+ "type": "prometheus",
790
+ "uid": "${DS_PROMETHEUS}"
791
+ },
792
+ "editorMode": "code",
793
+ "expr": "rate(vllm:time_to_first_token_seconds_sum[$__rate_interval])\n/\nrate(vllm:time_to_first_token_seconds_count[$__rate_interval])",
794
+ "hide": false,
795
+ "instant": false,
796
+ "legendFormat": "Average",
797
+ "range": true,
798
+ "refId": "E"
799
+ }
800
+ ],
801
+ "title": "Time To First Token Latency",
802
+ "type": "timeseries"
803
+ },
804
+ {
805
+ "datasource": {
806
+ "type": "prometheus",
807
+ "uid": "${DS_PROMETHEUS}"
808
+ },
809
+ "description": "Percentage of used cache blocks by vLLM.",
810
+ "fieldConfig": {
811
+ "defaults": {
812
+ "color": {
813
+ "mode": "palette-classic"
814
+ },
815
+ "custom": {
816
+ "axisBorderShow": false,
817
+ "axisCenteredZero": false,
818
+ "axisColorMode": "text",
819
+ "axisLabel": "",
820
+ "axisPlacement": "auto",
821
+ "barAlignment": 0,
822
+ "drawStyle": "line",
823
+ "fillOpacity": 0,
824
+ "gradientMode": "none",
825
+ "hideFrom": {
826
+ "legend": false,
827
+ "tooltip": false,
828
+ "viz": false
829
+ },
830
+ "insertNulls": false,
831
+ "lineInterpolation": "linear",
832
+ "lineWidth": 1,
833
+ "pointSize": 5,
834
+ "scaleDistribution": {
835
+ "type": "linear"
836
+ },
837
+ "showPoints": "auto",
838
+ "spanNulls": false,
839
+ "stacking": {
840
+ "group": "A",
841
+ "mode": "none"
842
+ },
843
+ "thresholdsStyle": {
844
+ "mode": "off"
845
+ }
846
+ },
847
+ "mappings": [],
848
+ "thresholds": {
849
+ "mode": "absolute",
850
+ "steps": [
851
+ {
852
+ "color": "green",
853
+ "value": null
854
+ },
855
+ {
856
+ "color": "red",
857
+ "value": 80
858
+ }
859
+ ]
860
+ },
861
+ "unit": "percentunit"
862
+ },
863
+ "overrides": []
864
+ },
865
+ "gridPos": {
866
+ "h": 8,
867
+ "w": 12,
868
+ "x": 12,
869
+ "y": 16
870
+ },
871
+ "id": 4,
872
+ "options": {
873
+ "legend": {
874
+ "calcs": [],
875
+ "displayMode": "list",
876
+ "placement": "bottom",
877
+ "showLegend": true
878
+ },
879
+ "tooltip": {
880
+ "mode": "single",
881
+ "sort": "none"
882
+ }
883
+ },
884
+ "targets": [
885
+ {
886
+ "datasource": {
887
+ "type": "prometheus",
888
+ "uid": "${DS_PROMETHEUS}"
889
+ },
890
+ "editorMode": "code",
891
+ "expr": "vllm:gpu_cache_usage_perc",
892
+ "instant": false,
893
+ "legendFormat": "GPU Cache Usage",
894
+ "range": true,
895
+ "refId": "A"
896
+ },
897
+ {
898
+ "datasource": {
899
+ "type": "prometheus",
900
+ "uid": "${DS_PROMETHEUS}"
901
+ },
902
+ "editorMode": "code",
903
+ "expr": "vllm:cpu_cache_usage_perc",
904
+ "hide": false,
905
+ "instant": false,
906
+ "legendFormat": "CPU Cache Usage",
907
+ "range": true,
908
+ "refId": "B"
909
+ }
910
+ ],
911
+ "title": "Cache Utilization",
912
+ "type": "timeseries"
913
+ }
914
+ ],
915
+ "refresh": "",
916
+ "schemaVersion": 39,
917
+ "tags": [],
918
+ "templating": {
919
+ "list": []
920
+ },
921
+ "time": {
922
+ "from": "now-5m",
923
+ "to": "now"
924
+ },
925
+ "timepicker": {},
926
+ "timezone": "",
927
+ "title": "vLLM",
928
+ "uid": "b281712d-8bff-41ef-9f3f-71ad43c05e9b",
929
+ "version": 2,
930
+ "weekStart": ""
931
+}
examples/production_monitoring/prometheus.yaml
new
+10
@@ -0,0 +1,10 @@
1
+# prometheus.yaml
2
+global:
3
+ scrape_interval: 5s
4
+ evaluation_interval: 30s
5
+
6
+scrape_configs:
7
+ - job_name: vllm
8
+ static_configs:
9
+ - targets:
10
+ - 'host.docker.internal:8000'
vllm/engine/llm_engine.py
+69
-79
@@ -10,7 +10,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
10
SchedulerConfig, LoRAConfig)
11
from vllm.core.scheduler import Scheduler, SchedulerOutputs
12
from vllm.engine.arg_utils import EngineArgs
13
-from vllm.engine.metrics import record_metrics
13
+from vllm.engine.metrics import StatLogger, Stats
14
from vllm.engine.ray_utils import RayWorkerVllm, initialize_cluster, ray
15
from vllm.logger import init_logger
16
from vllm.outputs import RequestOutput
@@ -28,8 +28,7 @@ if TYPE_CHECKING:
28
from ray.util.placement_group import PlacementGroup
29
30
logger = init_logger(__name__)
31
-
32
-_LOGGING_INTERVAL_SEC = 5
31
+_LOCAL_LOGGING_INTERVAL_SEC = 5
32
33
34
class LLMEngine:
@@ -116,12 +115,10 @@ class LLMEngine:
115
# Create the scheduler.
116
self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
117
119
- # Logging.
120
- self.last_logging_time = 0.0
121
- # List of (timestamp, num_tokens)
122
- self.num_prompt_tokens: List[Tuple[float, int]] = []
123
- # List of (timestamp, num_tokens)
124
- self.num_generation_tokens: List[Tuple[float, int]] = []
118
+ # Metric Logging.
119
+ if self.log_stats:
120
+ self.stat_logger = StatLogger(
121
+ local_interval=_LOCAL_LOGGING_INTERVAL_SEC)
122
123
def get_tokenizer_for_seq(self, sequence: Sequence):
124
return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
@@ -537,6 +534,7 @@ class LLMEngine:
534
535
def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
536
outputs: SequenceGroupOutput) -> None:
537
+
538
# Process prompt logprobs
539
prompt_logprobs = outputs.prompt_logprobs
540
if prompt_logprobs is not None:
@@ -732,10 +730,10 @@ class LLMEngine:
730
and not seq_group.prefix.computed):
731
seq_group.prefix.computed = True
732
733
+ # Log stats.
734
if self.log_stats:
736
- # Log the system stats.
737
- self._log_system_stats(scheduler_outputs.prompt_run,
738
- scheduler_outputs.num_batched_tokens)
735
+ self.stat_logger.log(self._get_stats(scheduler_outputs))
736
+
737
return request_outputs
738
739
def step(self) -> List[RequestOutput]:
@@ -810,81 +808,73 @@ class LLMEngine:
808
return self._process_model_outputs(output, scheduler_outputs)
809
810
def do_log_stats(self) -> None:
813
- self._log_system_stats(False, 0)
811
+ """Forced log when no requests active."""
812
+ if self.log_stats:
813
+ self.stat_logger.log(self._get_stats(scheduler_outputs=None))
814
815
- def _log_system_stats(
816
- self,
817
- prompt_run: bool,
818
- num_batched_tokens: int,
819
- ) -> None:
815
+ def _get_stats(self,
816
+ scheduler_outputs: Optional[SchedulerOutputs]) -> Stats:
817
+ """Get Stats to be Logged to Prometheus."""
818
now = time.monotonic()
821
- # Log the number of batched input tokens.
822
- if prompt_run:
823
- self.num_prompt_tokens.append((now, num_batched_tokens))
824
- else:
825
- self.num_generation_tokens.append((now, num_batched_tokens))
819
827
- should_log = now - self.last_logging_time >= _LOGGING_INTERVAL_SEC
828
- if not should_log:
829
- return
820
+ # KV Cache Usage in %.
821
+ num_total_gpu = self.cache_config.num_gpu_blocks
822
+ num_free_gpu = self.scheduler.block_manager.get_num_free_gpu_blocks()
823
+ gpu_cache_usage = 1.0 - (num_free_gpu / num_total_gpu)
824
831
- # Discard the old stats.
832
- self.num_prompt_tokens = [(t, n) for t, n in self.num_prompt_tokens
833
- if now - t < _LOGGING_INTERVAL_SEC]
834
- self.num_generation_tokens = [(t, n)
835
- for t, n in self.num_generation_tokens
836
- if now - t < _LOGGING_INTERVAL_SEC]
837
-
838
- if len(self.num_prompt_tokens) > 1:
839
- total_num_tokens = sum(n for _, n in self.num_prompt_tokens[:-1])
840
- window = now - self.num_prompt_tokens[0][0]
841
- avg_prompt_throughput = total_num_tokens / window
842
- else:
843
- avg_prompt_throughput = 0.0
844
- if len(self.num_generation_tokens) > 1:
845
- total_num_tokens = sum(n
846
- for _, n in self.num_generation_tokens[:-1])
847
- window = now - self.num_generation_tokens[0][0]
848
- avg_generation_throughput = total_num_tokens / window
849
- else:
850
- avg_generation_throughput = 0.0
851
-
852
- total_num_gpu_blocks = self.cache_config.num_gpu_blocks
853
- num_free_gpu_blocks = (
854
- self.scheduler.block_manager.get_num_free_gpu_blocks())
855
- num_used_gpu_blocks = total_num_gpu_blocks - num_free_gpu_blocks
856
- gpu_cache_usage = num_used_gpu_blocks / total_num_gpu_blocks
857
-
858
- total_num_cpu_blocks = self.cache_config.num_cpu_blocks
859
- if total_num_cpu_blocks > 0:
860
- num_free_cpu_blocks = (
861
- self.scheduler.block_manager.get_num_free_cpu_blocks())
862
- num_used_cpu_blocks = total_num_cpu_blocks - num_free_cpu_blocks
863
- cpu_cache_usage = num_used_cpu_blocks / total_num_cpu_blocks
864
- else:
865
- cpu_cache_usage = 0.0
866
-
867
- record_metrics(
868
- avg_prompt_throughput=avg_prompt_throughput,
869
- avg_generation_throughput=avg_generation_throughput,
870
- scheduler_running=len(self.scheduler.running),
871
- scheduler_swapped=len(self.scheduler.swapped),
872
- scheduler_waiting=len(self.scheduler.waiting),
825
+ num_total_cpu = self.cache_config.num_cpu_blocks
826
+ cpu_cache_usage = 0.
827
+ if num_total_cpu > 0:
828
+ num_free_cpu = self.scheduler.block_manager.get_num_free_cpu_blocks(
829
+ )
830
+ cpu_cache_usage = 1.0 - (num_free_cpu / num_total_cpu)
831
+
832
+ # Scheduler State
833
+ num_running = len(self.scheduler.running)
834
+ num_swapped = len(self.scheduler.swapped)
835
+ num_waiting = len(self.scheduler.waiting)
836
+
837
+ # Iteration stats if we have scheduler output.
838
+ num_prompt_tokens = 0
839
+ num_generation_tokens = 0
840
+ time_to_first_tokens = []
841
+ time_per_output_tokens = []
842
+ time_e2e_requests = []
843
+ if scheduler_outputs is not None:
844
+ prompt_run = scheduler_outputs.prompt_run
845
+
846
+ # Number of Tokens.
847
+ if prompt_run:
848
+ num_prompt_tokens = scheduler_outputs.num_batched_tokens
849
+ else:
850
+ num_generation_tokens = scheduler_outputs.num_batched_tokens
851
+
852
+ # Latency Timings.
853
+ time_last_iters = []
854
+ for seq_group in scheduler_outputs.scheduled_seq_groups:
855
+ # Time since last token. (n.b. updates seq_group.last_token_time)
856
+ time_last_iters.append(seq_group.get_last_latency(now))
857
+ # Time since arrival for all finished requests.
858
+ if seq_group.is_finished():
859
+ time_e2e_requests.append(now - seq_group.arrival_time)
860
+
861
+ time_to_first_tokens = time_last_iters if prompt_run else []
862
+ time_per_output_tokens = [] if prompt_run else time_last_iters
863
+
864
+ return Stats(
865
+ now=now,
866
+ num_running=num_running,
867
+ num_swapped=num_swapped,
868
+ num_waiting=num_waiting,
869
gpu_cache_usage=gpu_cache_usage,
870
cpu_cache_usage=cpu_cache_usage,
871
+ num_prompt_tokens=num_prompt_tokens,
872
+ num_generation_tokens=num_generation_tokens,
873
+ time_to_first_tokens=time_to_first_tokens,
874
+ time_per_output_tokens=time_per_output_tokens,
875
+ time_e2e_requests=time_e2e_requests,
876
)
877
877
- logger.info("Avg prompt throughput: "
878
- f"{avg_prompt_throughput:.1f} tokens/s, "
879
- "Avg generation throughput: "
880
- f"{avg_generation_throughput:.1f} tokens/s, "
881
- f"Running: {len(self.scheduler.running)} reqs, "
882
- f"Swapped: {len(self.scheduler.swapped)} reqs, "
883
- f"Pending: {len(self.scheduler.waiting)} reqs, "
884
- f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
885
- f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
886
- self.last_logging_time = now
887
-
878
def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
879
"""Decodes the new token for a sequence."""
880
(new_tokens, new_output_text, prefix_offset,
vllm/engine/metrics.py
+144
-22
@@ -1,4 +1,19 @@
1
-from aioprometheus import Gauge
1
+from vllm.logger import init_logger
2
+from aioprometheus import Counter, Gauge, Histogram
3
+
4
+import time
5
+import numpy as np
6
+from typing import List
7
+from dataclasses import dataclass
8
+
9
+logger = init_logger(__name__)
10
+
11
+labels = {}
12
+
13
+
14
+def add_global_metrics_labels(**kwargs):
15
+ labels.update(kwargs)
16
+
17
18
# The begin-* and end* here are used by the documentation generator
19
# to extract the metrics definitions.
@@ -9,12 +24,16 @@ gauge_avg_prompt_throughput = Gauge("vllm:avg_prompt_throughput_toks_per_s",
24
gauge_avg_generation_throughput = Gauge(
25
"vllm:avg_generation_throughput_toks_per_s",
26
"Average generation throughput in tokens/s.")
27
+counter_prompt_tokens = Counter("vllm:prompt_tokens_total",
28
+ "Number of prefill tokens processed.")
29
+counter_generation_tokens = Counter("vllm:generation_tokens_total",
30
+ "Number of generation tokens processed.")
31
32
gauge_scheduler_running = Gauge(
33
"vllm:num_requests_running",
15
- "Number of requests that is currently running for inference.")
34
+ "Number of requests currently running on GPU.")
35
gauge_scheduler_swapped = Gauge("vllm:num_requests_swapped",
17
- "Number requests swapped to CPU.")
36
+ "Number of requests swapped to CPU.")
37
gauge_scheduler_waiting = Gauge("vllm:num_requests_waiting",
38
"Number of requests waiting to be processed.")
39
@@ -24,28 +43,131 @@ gauge_gpu_cache_usage = Gauge(
43
gauge_cpu_cache_usage = Gauge(
44
"vllm:cpu_cache_usage_perc",
45
"CPU KV-cache usage. 1 means 100 percent usage.")
46
+
47
+histogram_time_to_first_token = Histogram(
48
+ "vllm:time_to_first_token_seconds",
49
+ "Histogram of time to first token in seconds.",
50
+ buckets=[
51
+ 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5, 0.75, 1.0,
52
+ 2.5, 5.0, 7.5, 10.0
53
+ ])
54
+histogram_time_per_output_tokens = Histogram(
55
+ "vllm:time_per_output_token_seconds",
56
+ "Histogram of time per output token in seconds.",
57
+ buckets=[
58
+ 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 2.5
59
+ ])
60
+histogram_e2e_request_latency = Histogram(
61
+ "vllm:e2e_request_latency_seconds",
62
+ "Histogram of end to end request latency in seconds.",
63
+ buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
64
# end-metrics-definitions
65
29
-labels = {}
66
67
+@dataclass
68
+class Stats:
69
+ """Created by LLMEngine for use by StatLogger."""
70
+ now: float
71
32
-def add_global_metrics_labels(**kwargs):
33
- labels.update(kwargs)
72
+ # System stats.
73
+ num_running: int
74
+ num_waiting: int
75
+ num_swapped: int
76
+ gpu_cache_usage: float
77
+ cpu_cache_usage: float
78
+
79
+ # Raw stats from last model iteration.
80
+ num_prompt_tokens: int
81
+ num_generation_tokens: int
82
+ time_to_first_tokens: List[float]
83
+ time_per_output_tokens: List[float]
84
+ time_e2e_requests: List[float]
85
+
86
+
87
+class StatLogger:
88
+ """StatLogger is used LLMEngine to log to Promethus and Stdout."""
89
+
90
+ def __init__(self, local_interval: float) -> None:
91
+ # Metadata for logging locally.
92
+ self.last_local_log = time.monotonic()
93
+ self.local_interval = local_interval
94
+
95
+ # Tracked stats over current local logging interval.
96
+ self.num_prompt_tokens: List[int] = []
97
+ self.num_generation_tokens: List[int] = []
98
+
99
+ def _get_throughput(self, tracked_stats: List[int], now: float) -> float:
100
+ return float(np.sum(tracked_stats) / (now - self.last_local_log))
101
+
102
+ def _local_interval_elapsed(self, now: float) -> bool:
103
+ elapsed_time = now - self.last_local_log
104
+ return elapsed_time > self.local_interval
105
+
106
+ def _log_prometheus(self, stats: Stats) -> None:
107
+ # Set system stat gauges.
108
+ gauge_scheduler_running.set(labels, stats.num_running)
109
+ gauge_scheduler_swapped.set(labels, stats.num_swapped)
110
+ gauge_scheduler_waiting.set(labels, stats.num_waiting)
111
+ gauge_gpu_cache_usage.set(labels, stats.gpu_cache_usage)
112
+ gauge_cpu_cache_usage.set(labels, stats.cpu_cache_usage)
113
+
114
+ # Add to token counters.
115
+ counter_prompt_tokens.add(labels, stats.num_prompt_tokens)
116
+ counter_generation_tokens.add(labels, stats.num_generation_tokens)
117
+
118
+ # Observe request level latencies in histograms.
119
+ for ttft in stats.time_to_first_tokens:
120
+ histogram_time_to_first_token.observe(labels, ttft)
121
+ for tpot in stats.time_per_output_tokens:
122
+ histogram_time_per_output_tokens.observe(labels, tpot)
123
+ for e2e in stats.time_e2e_requests:
124
+ histogram_e2e_request_latency.observe(labels, e2e)
125
+
126
+ def _log_prometheus_interval(self, prompt_throughput: float,
127
+ generation_throughput: float) -> None:
128
+ # Logs metrics to prometheus that are computed every logging_interval.
129
+ # Support legacy gauge metrics that make throughput calculations on the vLLM side.
130
+ # Moving forward, we should use counters like counter_prompt_tokens, counter_generation_tokens
131
+ # Which log raw data and calculate summaries using rate() on the grafana/prometheus side.
132
+ # See https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666
133
+ gauge_avg_prompt_throughput.set(labels, prompt_throughput)
134
+ gauge_avg_generation_throughput.set(labels, generation_throughput)
135
+
136
+ def log(self, stats: Stats) -> None:
137
+ """Called by LLMEngine.
138
+ Logs to prometheus and tracked stats every iteration.
139
+ Logs to Stdout every self.local_interval seconds."""
140
+
141
+ # Log to prometheus.
142
+ self._log_prometheus(stats)
143
+
144
+ # Save tracked stats for token counters.
145
+ self.num_prompt_tokens.append(stats.num_prompt_tokens)
146
+ self.num_generation_tokens.append(stats.num_generation_tokens)
147
+
148
+ # Log locally every local_interval seconds.
149
+ if self._local_interval_elapsed(stats.now):
150
+
151
+ # Compute summary metrics for tracked stats (and log them to promethus if applicable).
152
+ prompt_throughput = self._get_throughput(self.num_prompt_tokens,
153
+ now=stats.now)
154
+ generation_throughput = self._get_throughput(
155
+ self.num_generation_tokens, now=stats.now)
156
+ self._log_prometheus_interval(
157
+ prompt_throughput=prompt_throughput,
158
+ generation_throughput=generation_throughput)
159
160
+ # Log to stdout.
161
+ logger.info(
162
+ f"Avg prompt throughput: {prompt_throughput:.1f} tokens/s, "
163
+ f"Avg generation throughput: {generation_throughput:.1f} tokens/s, "
164
+ f"Running: {stats.num_running} reqs, "
165
+ f"Swapped: {stats.num_swapped} reqs, "
166
+ f"Pending: {stats.num_waiting} reqs, "
167
+ f"GPU KV cache usage: {stats.gpu_cache_usage * 100:.1f}%, "
168
+ f"CPU KV cache usage: {stats.cpu_cache_usage * 100:.1f}%")
169
36
-def record_metrics(
37
- avg_prompt_throughput: float,
38
- avg_generation_throughput: float,
39
- scheduler_running: int,
40
- scheduler_swapped: int,
41
- scheduler_waiting: int,
42
- gpu_cache_usage: float,
43
- cpu_cache_usage: float,
44
-):
45
- gauge_avg_prompt_throughput.set(labels, avg_prompt_throughput)
46
- gauge_avg_generation_throughput.set(labels, avg_generation_throughput)
47
- gauge_scheduler_running.set(labels, scheduler_running)
48
- gauge_scheduler_swapped.set(labels, scheduler_swapped)
49
- gauge_scheduler_waiting.set(labels, scheduler_waiting)
50
- gauge_gpu_cache_usage.set(labels, gpu_cache_usage)
51
- gauge_cpu_cache_usage.set(labels, cpu_cache_usage)
170
+ # Reset tracked stats for next interval.
171
+ self.num_prompt_tokens = []
172
+ self.num_generation_tokens = []
173
+ self.last_local_log = stats.now
vllm/sequence.py
+7
-1
@@ -52,7 +52,6 @@ class SequenceStatus(enum.Enum):
52
class SequenceData:
53
"""Data associated with a sequence.
54
55
-
55
Args:
56
prompt_token_ids: The token IDs of the prompt.
57
@@ -254,6 +253,7 @@ class SequenceGroup:
253
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
254
self.sampling_params = sampling_params
255
self.arrival_time = arrival_time
256
+ self.last_token_time = arrival_time
257
self.lora_request = lora_request
258
self.prefix: Optional[Prefix] = prefix
259
self.prompt_logprobs: Optional[PromptLogprobs] = None
@@ -274,6 +274,12 @@ class SequenceGroup:
274
def lora_int_id(self) -> int:
275
return self.lora_request.lora_int_id if self.lora_request else 0
276
277
+ def get_last_latency(self, now: float) -> float:
278
+ """Gets last token latency for Request level timings."""
279
+ latency = now - self.last_token_time
280
+ self.last_token_time = now
281
+ return latency
282
+
283
def get_max_num_running_seqs(self) -> int:
284
"""The maximum number of sequences running in parallel in the remaining
285
lifetime of the request."""