A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
allow user to chose which vllm's merics to display in grafana (#3393)
Allen.Dou committed
Mar 14, 2024 at 14:35 UTC
a37415c31b3b5c7ab40d2d897192025f0ca7be08
1 file changed
+88
-96
examples/production_monitoring/grafana.json
+88
-96
@@ -1,35 +1,4 @@
1
{
2
- "__inputs": [
3
- {
4
- "name": "DS_PROMETHEUS",
5
- "label": "prometheus",
6
- "description": "",
7
- "type": "datasource",
8
- "pluginId": "prometheus",
9
- "pluginName": "Prometheus"
10
- }
11
- ],
12
- "__elements": {},
13
- "__requires": [
14
- {
15
- "type": "grafana",
16
- "id": "grafana",
17
- "name": "Grafana",
18
- "version": "10.2.3"
19
- },
20
- {
21
- "type": "datasource",
22
- "id": "prometheus",
23
- "name": "Prometheus",
24
- "version": "1.0.0"
25
- },
26
- {
27
- "type": "panel",
28
- "id": "timeseries",
29
- "name": "Time series",
30
- "version": ""
31
- }
32
- ],
2
"annotations": {
3
"list": [
4
{
@@ -42,6 +11,12 @@
11
"hide": true,
12
"iconColor": "rgba(0, 211, 255, 1)",
13
"name": "Annotations & Alerts",
14
+ "target": {
15
+ "limit": 100,
16
+ "matchAny": false,
17
+ "tags": [],
18
+ "type": "dashboard"
19
+ },
20
"type": "dashboard"
21
}
22
]
@@ -50,14 +25,14 @@
25
"editable": true,
26
"fiscalYearStartMonth": 0,
27
"graphTooltip": 0,
53
- "id": null,
28
+ "id": 29,
29
"links": [],
30
"liveNow": false,
31
"panels": [
32
{
33
"datasource": {
34
"type": "prometheus",
60
- "uid": "${DS_PROMETHEUS}"
35
+ "uid": "prometheus"
36
},
37
"description": "End to end request latency measured in seconds.",
38
"fieldConfig": {
@@ -66,7 +41,6 @@
41
"mode": "palette-classic"
42
},
43
"custom": {
69
- "axisBorderShow": false,
44
"axisCenteredZero": false,
45
"axisColorMode": "text",
46
"axisLabel": "",
@@ -80,7 +54,6 @@
54
"tooltip": false,
55
"viz": false
56
},
83
- "insertNulls": false,
57
"lineInterpolation": "linear",
58
"lineWidth": 1,
59
"pointSize": 5,
@@ -138,11 +111,11 @@
111
{
112
"datasource": {
113
"type": "prometheus",
141
- "uid": "${DS_PROMETHEUS}"
114
+ "uid": "prometheus"
115
},
116
"disableTextWrap": false,
117
"editorMode": "builder",
145
- "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
118
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
119
"fullMetaSearch": false,
120
"includeNullMetadata": false,
121
"instant": false,
@@ -154,11 +127,11 @@
127
{
128
"datasource": {
129
"type": "prometheus",
157
- "uid": "${DS_PROMETHEUS}"
130
+ "uid": "prometheus"
131
},
132
"disableTextWrap": false,
133
"editorMode": "builder",
161
- "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
134
+ "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
135
"fullMetaSearch": false,
136
"hide": false,
137
"includeNullMetadata": false,
@@ -171,11 +144,11 @@
144
{
145
"datasource": {
146
"type": "prometheus",
174
- "uid": "${DS_PROMETHEUS}"
147
+ "uid": "prometheus"
148
},
149
"disableTextWrap": false,
150
"editorMode": "builder",
178
- "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
151
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
152
"fullMetaSearch": false,
153
"hide": false,
154
"includeNullMetadata": false,
@@ -188,11 +161,11 @@
161
{
162
"datasource": {
163
"type": "prometheus",
191
- "uid": "${DS_PROMETHEUS}"
164
+ "uid": "prometheus"
165
},
166
"disableTextWrap": false,
167
"editorMode": "builder",
195
- "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
168
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
169
"fullMetaSearch": false,
170
"hide": false,
171
"includeNullMetadata": false,
@@ -205,10 +178,10 @@
178
{
179
"datasource": {
180
"type": "prometheus",
208
- "uid": "${DS_PROMETHEUS}"
181
+ "uid": "prometheus"
182
},
183
"editorMode": "code",
211
- "expr": "rate(vllm:e2e_request_latency_seconds_sum[$__rate_interval])\n/\nrate(vllm:e2e_request_latency_seconds_count[$__rate_interval])",
184
+ "expr": "rate(vllm:e2e_request_latency_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:e2e_request_latency_seconds_count{model_name=\"$model_name\"}[$__rate_interval])",
185
"hide": false,
186
"instant": false,
187
"legendFormat": "Average",
@@ -222,7 +195,7 @@
195
{
196
"datasource": {
197
"type": "prometheus",
225
- "uid": "${DS_PROMETHEUS}"
198
+ "uid": "prometheus"
199
},
200
"description": "Number of tokens processed per second",
201
"fieldConfig": {
@@ -231,7 +204,6 @@
204
"mode": "palette-classic"
205
},
206
"custom": {
234
- "axisBorderShow": false,
207
"axisCenteredZero": false,
208
"axisColorMode": "text",
209
"axisLabel": "",
@@ -245,7 +217,6 @@
217
"tooltip": false,
218
"viz": false
219
},
248
- "insertNulls": false,
220
"lineInterpolation": "linear",
221
"lineWidth": 1,
222
"pointSize": 5,
@@ -302,11 +273,11 @@
273
{
274
"datasource": {
275
"type": "prometheus",
305
- "uid": "${DS_PROMETHEUS}"
276
+ "uid": "prometheus"
277
},
278
"disableTextWrap": false,
279
"editorMode": "builder",
309
- "expr": "rate(vllm:prompt_tokens_total[$__rate_interval])",
280
+ "expr": "rate(vllm:prompt_tokens_total{model_name=\"$model_name\"}[$__rate_interval])",
281
"fullMetaSearch": false,
282
"includeNullMetadata": false,
283
"instant": false,
@@ -318,11 +289,11 @@
289
{
290
"datasource": {
291
"type": "prometheus",
321
- "uid": "${DS_PROMETHEUS}"
292
+ "uid": "prometheus"
293
},
294
"disableTextWrap": false,
295
"editorMode": "builder",
325
- "expr": "rate(vllm:generation_tokens_total[$__rate_interval])",
296
+ "expr": "rate(vllm:generation_tokens_total{model_name=\"$model_name\"}[$__rate_interval])",
297
"fullMetaSearch": false,
298
"hide": false,
299
"includeNullMetadata": false,
@@ -339,7 +310,7 @@
310
{
311
"datasource": {
312
"type": "prometheus",
342
- "uid": "${DS_PROMETHEUS}"
313
+ "uid": "prometheus"
314
},
315
"description": "Inter token latency in seconds.",
316
"fieldConfig": {
@@ -348,7 +319,6 @@
319
"mode": "palette-classic"
320
},
321
"custom": {
351
- "axisBorderShow": false,
322
"axisCenteredZero": false,
323
"axisColorMode": "text",
324
"axisLabel": "",
@@ -362,7 +332,6 @@
332
"tooltip": false,
333
"viz": false
334
},
365
- "insertNulls": false,
335
"lineInterpolation": "linear",
336
"lineWidth": 1,
337
"pointSize": 5,
@@ -420,11 +389,11 @@
389
{
390
"datasource": {
391
"type": "prometheus",
423
- "uid": "${DS_PROMETHEUS}"
392
+ "uid": "prometheus"
393
},
394
"disableTextWrap": false,
395
"editorMode": "builder",
427
- "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
396
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
397
"fullMetaSearch": false,
398
"includeNullMetadata": false,
399
"instant": false,
@@ -436,11 +405,11 @@
405
{
406
"datasource": {
407
"type": "prometheus",
439
- "uid": "${DS_PROMETHEUS}"
408
+ "uid": "prometheus"
409
},
410
"disableTextWrap": false,
411
"editorMode": "builder",
443
- "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
412
+ "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
413
"fullMetaSearch": false,
414
"hide": false,
415
"includeNullMetadata": false,
@@ -453,11 +422,11 @@
422
{
423
"datasource": {
424
"type": "prometheus",
456
- "uid": "${DS_PROMETHEUS}"
425
+ "uid": "prometheus"
426
},
427
"disableTextWrap": false,
428
"editorMode": "builder",
460
- "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
429
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
430
"fullMetaSearch": false,
431
"hide": false,
432
"includeNullMetadata": false,
@@ -470,11 +439,11 @@
439
{
440
"datasource": {
441
"type": "prometheus",
473
- "uid": "${DS_PROMETHEUS}"
442
+ "uid": "prometheus"
443
},
444
"disableTextWrap": false,
445
"editorMode": "builder",
477
- "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
446
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
447
"fullMetaSearch": false,
448
"hide": false,
449
"includeNullMetadata": false,
@@ -487,10 +456,10 @@
456
{
457
"datasource": {
458
"type": "prometheus",
490
- "uid": "${DS_PROMETHEUS}"
459
+ "uid": "prometheus"
460
},
461
"editorMode": "code",
493
- "expr": "rate(vllm:time_per_output_token_seconds_sum[$__rate_interval])\n/\nrate(vllm:time_per_output_token_seconds_count[$__rate_interval])",
462
+ "expr": "rate(vllm:time_per_output_token_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:time_per_output_token_seconds_count{model_name=\"$model_name\"}[$__rate_interval])",
463
"hide": false,
464
"instant": false,
465
"legendFormat": "Mean",
@@ -504,7 +473,7 @@
473
{
474
"datasource": {
475
"type": "prometheus",
507
- "uid": "${DS_PROMETHEUS}"
476
+ "uid": "prometheus"
477
},
478
"description": "Number of requests in RUNNING, WAITING, and SWAPPED state",
479
"fieldConfig": {
@@ -513,7 +482,6 @@
482
"mode": "palette-classic"
483
},
484
"custom": {
516
- "axisBorderShow": false,
485
"axisCenteredZero": false,
486
"axisColorMode": "text",
487
"axisLabel": "",
@@ -527,7 +495,6 @@
495
"tooltip": false,
496
"viz": false
497
},
530
- "insertNulls": false,
498
"lineInterpolation": "linear",
499
"lineWidth": 1,
500
"pointSize": 5,
@@ -585,11 +552,11 @@
552
{
553
"datasource": {
554
"type": "prometheus",
588
- "uid": "${DS_PROMETHEUS}"
555
+ "uid": "prometheus"
556
},
557
"disableTextWrap": false,
558
"editorMode": "builder",
592
- "expr": "vllm:num_requests_running",
559
+ "expr": "vllm:num_requests_running{model_name=\"$model_name\"}",
560
"fullMetaSearch": false,
561
"includeNullMetadata": true,
562
"instant": false,
@@ -601,11 +568,11 @@
568
{
569
"datasource": {
570
"type": "prometheus",
604
- "uid": "${DS_PROMETHEUS}"
571
+ "uid": "prometheus"
572
},
573
"disableTextWrap": false,
574
"editorMode": "builder",
608
- "expr": "vllm:num_requests_swapped",
575
+ "expr": "vllm:num_requests_swapped{model_name=\"$model_name\"}",
576
"fullMetaSearch": false,
577
"hide": false,
578
"includeNullMetadata": true,
@@ -618,11 +585,11 @@
585
{
586
"datasource": {
587
"type": "prometheus",
621
- "uid": "${DS_PROMETHEUS}"
588
+ "uid": "prometheus"
589
},
590
"disableTextWrap": false,
591
"editorMode": "builder",
625
- "expr": "vllm:num_requests_waiting",
592
+ "expr": "vllm:num_requests_waiting{model_name=\"$model_name\"}",
593
"fullMetaSearch": false,
594
"hide": false,
595
"includeNullMetadata": true,
@@ -639,7 +606,7 @@
606
{
607
"datasource": {
608
"type": "prometheus",
642
- "uid": "${DS_PROMETHEUS}"
609
+ "uid": "prometheus"
610
},
611
"description": "P50, P90, P95, and P99 TTFT latency in seconds.",
612
"fieldConfig": {
@@ -648,7 +615,6 @@
615
"mode": "palette-classic"
616
},
617
"custom": {
651
- "axisBorderShow": false,
618
"axisCenteredZero": false,
619
"axisColorMode": "text",
620
"axisLabel": "",
@@ -662,7 +628,6 @@
628
"tooltip": false,
629
"viz": false
630
},
665
- "insertNulls": false,
631
"lineInterpolation": "linear",
632
"lineWidth": 1,
633
"pointSize": 5,
@@ -720,11 +685,11 @@
685
{
686
"datasource": {
687
"type": "prometheus",
723
- "uid": "${DS_PROMETHEUS}"
688
+ "uid": "prometheus"
689
},
690
"disableTextWrap": false,
691
"editorMode": "builder",
727
- "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
692
+ "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
693
"fullMetaSearch": false,
694
"hide": false,
695
"includeNullMetadata": false,
@@ -737,11 +702,11 @@
702
{
703
"datasource": {
704
"type": "prometheus",
740
- "uid": "${DS_PROMETHEUS}"
705
+ "uid": "prometheus"
706
},
707
"disableTextWrap": false,
708
"editorMode": "builder",
744
- "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
709
+ "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
710
"fullMetaSearch": false,
711
"includeNullMetadata": false,
712
"instant": false,
@@ -753,11 +718,11 @@
718
{
719
"datasource": {
720
"type": "prometheus",
756
- "uid": "${DS_PROMETHEUS}"
721
+ "uid": "prometheus"
722
},
723
"disableTextWrap": false,
724
"editorMode": "builder",
760
- "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
725
+ "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
726
"fullMetaSearch": false,
727
"hide": false,
728
"includeNullMetadata": false,
@@ -770,11 +735,11 @@
735
{
736
"datasource": {
737
"type": "prometheus",
773
- "uid": "${DS_PROMETHEUS}"
738
+ "uid": "prometheus"
739
},
740
"disableTextWrap": false,
741
"editorMode": "builder",
777
- "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
742
+ "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
743
"fullMetaSearch": false,
744
"hide": false,
745
"includeNullMetadata": false,
@@ -787,10 +752,10 @@
752
{
753
"datasource": {
754
"type": "prometheus",
790
- "uid": "${DS_PROMETHEUS}"
755
+ "uid": "prometheus"
756
},
757
"editorMode": "code",
793
- "expr": "rate(vllm:time_to_first_token_seconds_sum[$__rate_interval])\n/\nrate(vllm:time_to_first_token_seconds_count[$__rate_interval])",
758
+ "expr": "rate(vllm:time_to_first_token_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:time_to_first_token_seconds_count{model_name=\"$model_name\"}[$__rate_interval])",
759
"hide": false,
760
"instant": false,
761
"legendFormat": "Average",
@@ -804,7 +769,7 @@
769
{
770
"datasource": {
771
"type": "prometheus",
807
- "uid": "${DS_PROMETHEUS}"
772
+ "uid": "prometheus"
773
},
774
"description": "Percentage of used cache blocks by vLLM.",
775
"fieldConfig": {
@@ -813,7 +778,6 @@
778
"mode": "palette-classic"
779
},
780
"custom": {
816
- "axisBorderShow": false,
781
"axisCenteredZero": false,
782
"axisColorMode": "text",
783
"axisLabel": "",
@@ -827,7 +791,6 @@
791
"tooltip": false,
792
"viz": false
793
},
830
- "insertNulls": false,
794
"lineInterpolation": "linear",
795
"lineWidth": 1,
796
"pointSize": 5,
@@ -885,10 +848,10 @@
848
{
849
"datasource": {
850
"type": "prometheus",
888
- "uid": "${DS_PROMETHEUS}"
851
+ "uid": "prometheus"
852
},
853
"editorMode": "code",
891
- "expr": "vllm:gpu_cache_usage_perc",
854
+ "expr": "vllm:gpu_cache_usage_perc{model_name=\"$model_name\"}",
855
"instant": false,
856
"legendFormat": "GPU Cache Usage",
857
"range": true,
@@ -897,10 +860,10 @@
860
{
861
"datasource": {
862
"type": "prometheus",
900
- "uid": "${DS_PROMETHEUS}"
863
+ "uid": "prometheus"
864
},
865
"editorMode": "code",
903
- "expr": "vllm:cpu_cache_usage_perc",
866
+ "expr": "vllm:cpu_cache_usage_perc{model_name=\"$model_name\"}",
867
"hide": false,
868
"instant": false,
869
"legendFormat": "CPU Cache Usage",
@@ -913,10 +876,39 @@
876
}
877
],
878
"refresh": "",
916
- "schemaVersion": 39,
879
+ "schemaVersion": 37,
880
+ "style": "dark",
881
"tags": [],
882
"templating": {
919
- "list": []
883
+ "list": [
884
+ {
885
+ "current": {
886
+ "selected": false,
887
+ "text": "vllm",
888
+ "value": "vllm"
889
+ },
890
+ "datasource": {
891
+ "type": "prometheus",
892
+ "uid": "prometheus"
893
+ },
894
+ "definition": "label_values(model_name)",
895
+ "hide": 0,
896
+ "includeAll": false,
897
+ "label": "model_name",
898
+ "multi": false,
899
+ "name": "model_name",
900
+ "options": [],
901
+ "query": {
902
+ "query": "label_values(model_name)",
903
+ "refId": "StandardVariableQuery"
904
+ },
905
+ "refresh": 1,
906
+ "regex": "",
907
+ "skipUrlSync": false,
908
+ "sort": 0,
909
+ "type": "query"
910
+ }
911
+ ]
912
},
913
"time": {
914
"from": "now-5m",