@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a37415c31b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

allow user to chose which vllm's merics to display in grafana (#3393)

Allen.Dou committed Mar 14, 2024 at 14:35 UTC a37415c31b3b5c7ab40d2d897192025f0ca7be08
1 file changed +88 -96
examples/production_monitoring/grafana.json
+88 -96
@@ -1,35 +1,4 @@
1 {
2 - "__inputs": [
3 - {
4 - "name": "DS_PROMETHEUS",
5 - "label": "prometheus",
6 - "description": "",
7 - "type": "datasource",
8 - "pluginId": "prometheus",
9 - "pluginName": "Prometheus"
10 - }
11 - ],
12 - "__elements": {},
13 - "__requires": [
14 - {
15 - "type": "grafana",
16 - "id": "grafana",
17 - "name": "Grafana",
18 - "version": "10.2.3"
19 - },
20 - {
21 - "type": "datasource",
22 - "id": "prometheus",
23 - "name": "Prometheus",
24 - "version": "1.0.0"
25 - },
26 - {
27 - "type": "panel",
28 - "id": "timeseries",
29 - "name": "Time series",
30 - "version": ""
31 - }
32 - ],
2 "annotations": {
3 "list": [
4 {
@@ -42,6 +11,12 @@
11 "hide": true,
12 "iconColor": "rgba(0, 211, 255, 1)",
13 "name": "Annotations & Alerts",
14 + "target": {
15 + "limit": 100,
16 + "matchAny": false,
17 + "tags": [],
18 + "type": "dashboard"
19 + },
20 "type": "dashboard"
21 }
22 ]
@@ -50,14 +25,14 @@
25 "editable": true,
26 "fiscalYearStartMonth": 0,
27 "graphTooltip": 0,
53 - "id": null,
28 + "id": 29,
29 "links": [],
30 "liveNow": false,
31 "panels": [
32 {
33 "datasource": {
34 "type": "prometheus",
60 - "uid": "${DS_PROMETHEUS}"
35 + "uid": "prometheus"
36 },
37 "description": "End to end request latency measured in seconds.",
38 "fieldConfig": {
@@ -66,7 +41,6 @@
41 "mode": "palette-classic"
42 },
43 "custom": {
69 - "axisBorderShow": false,
44 "axisCenteredZero": false,
45 "axisColorMode": "text",
46 "axisLabel": "",
@@ -80,7 +54,6 @@
54 "tooltip": false,
55 "viz": false
56 },
83 - "insertNulls": false,
57 "lineInterpolation": "linear",
58 "lineWidth": 1,
59 "pointSize": 5,
@@ -138,11 +111,11 @@
111 {
112 "datasource": {
113 "type": "prometheus",
141 - "uid": "${DS_PROMETHEUS}"
114 + "uid": "prometheus"
115 },
116 "disableTextWrap": false,
117 "editorMode": "builder",
145 - "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
118 + "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
119 "fullMetaSearch": false,
120 "includeNullMetadata": false,
121 "instant": false,
@@ -154,11 +127,11 @@
127 {
128 "datasource": {
129 "type": "prometheus",
157 - "uid": "${DS_PROMETHEUS}"
130 + "uid": "prometheus"
131 },
132 "disableTextWrap": false,
133 "editorMode": "builder",
161 - "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
134 + "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
135 "fullMetaSearch": false,
136 "hide": false,
137 "includeNullMetadata": false,
@@ -171,11 +144,11 @@
144 {
145 "datasource": {
146 "type": "prometheus",
174 - "uid": "${DS_PROMETHEUS}"
147 + "uid": "prometheus"
148 },
149 "disableTextWrap": false,
150 "editorMode": "builder",
178 - "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
151 + "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
152 "fullMetaSearch": false,
153 "hide": false,
154 "includeNullMetadata": false,
@@ -188,11 +161,11 @@
161 {
162 "datasource": {
163 "type": "prometheus",
191 - "uid": "${DS_PROMETHEUS}"
164 + "uid": "prometheus"
165 },
166 "disableTextWrap": false,
167 "editorMode": "builder",
195 - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
168 + "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
169 "fullMetaSearch": false,
170 "hide": false,
171 "includeNullMetadata": false,
@@ -205,10 +178,10 @@
178 {
179 "datasource": {
180 "type": "prometheus",
208 - "uid": "${DS_PROMETHEUS}"
181 + "uid": "prometheus"
182 },
183 "editorMode": "code",
211 - "expr": "rate(vllm:e2e_request_latency_seconds_sum[$__rate_interval])\n/\nrate(vllm:e2e_request_latency_seconds_count[$__rate_interval])",
184 + "expr": "rate(vllm:e2e_request_latency_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:e2e_request_latency_seconds_count{model_name=\"$model_name\"}[$__rate_interval])",
185 "hide": false,
186 "instant": false,
187 "legendFormat": "Average",
@@ -222,7 +195,7 @@
195 {
196 "datasource": {
197 "type": "prometheus",
225 - "uid": "${DS_PROMETHEUS}"
198 + "uid": "prometheus"
199 },
200 "description": "Number of tokens processed per second",
201 "fieldConfig": {
@@ -231,7 +204,6 @@
204 "mode": "palette-classic"
205 },
206 "custom": {
234 - "axisBorderShow": false,
207 "axisCenteredZero": false,
208 "axisColorMode": "text",
209 "axisLabel": "",
@@ -245,7 +217,6 @@
217 "tooltip": false,
218 "viz": false
219 },
248 - "insertNulls": false,
220 "lineInterpolation": "linear",
221 "lineWidth": 1,
222 "pointSize": 5,
@@ -302,11 +273,11 @@
273 {
274 "datasource": {
275 "type": "prometheus",
305 - "uid": "${DS_PROMETHEUS}"
276 + "uid": "prometheus"
277 },
278 "disableTextWrap": false,
279 "editorMode": "builder",
309 - "expr": "rate(vllm:prompt_tokens_total[$__rate_interval])",
280 + "expr": "rate(vllm:prompt_tokens_total{model_name=\"$model_name\"}[$__rate_interval])",
281 "fullMetaSearch": false,
282 "includeNullMetadata": false,
283 "instant": false,
@@ -318,11 +289,11 @@
289 {
290 "datasource": {
291 "type": "prometheus",
321 - "uid": "${DS_PROMETHEUS}"
292 + "uid": "prometheus"
293 },
294 "disableTextWrap": false,
295 "editorMode": "builder",
325 - "expr": "rate(vllm:generation_tokens_total[$__rate_interval])",
296 + "expr": "rate(vllm:generation_tokens_total{model_name=\"$model_name\"}[$__rate_interval])",
297 "fullMetaSearch": false,
298 "hide": false,
299 "includeNullMetadata": false,
@@ -339,7 +310,7 @@
310 {
311 "datasource": {
312 "type": "prometheus",
342 - "uid": "${DS_PROMETHEUS}"
313 + "uid": "prometheus"
314 },
315 "description": "Inter token latency in seconds.",
316 "fieldConfig": {
@@ -348,7 +319,6 @@
319 "mode": "palette-classic"
320 },
321 "custom": {
351 - "axisBorderShow": false,
322 "axisCenteredZero": false,
323 "axisColorMode": "text",
324 "axisLabel": "",
@@ -362,7 +332,6 @@
332 "tooltip": false,
333 "viz": false
334 },
365 - "insertNulls": false,
335 "lineInterpolation": "linear",
336 "lineWidth": 1,
337 "pointSize": 5,
@@ -420,11 +389,11 @@
389 {
390 "datasource": {
391 "type": "prometheus",
423 - "uid": "${DS_PROMETHEUS}"
392 + "uid": "prometheus"
393 },
394 "disableTextWrap": false,
395 "editorMode": "builder",
427 - "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
396 + "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
397 "fullMetaSearch": false,
398 "includeNullMetadata": false,
399 "instant": false,
@@ -436,11 +405,11 @@
405 {
406 "datasource": {
407 "type": "prometheus",
439 - "uid": "${DS_PROMETHEUS}"
408 + "uid": "prometheus"
409 },
410 "disableTextWrap": false,
411 "editorMode": "builder",
443 - "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
412 + "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
413 "fullMetaSearch": false,
414 "hide": false,
415 "includeNullMetadata": false,
@@ -453,11 +422,11 @@
422 {
423 "datasource": {
424 "type": "prometheus",
456 - "uid": "${DS_PROMETHEUS}"
425 + "uid": "prometheus"
426 },
427 "disableTextWrap": false,
428 "editorMode": "builder",
460 - "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
429 + "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
430 "fullMetaSearch": false,
431 "hide": false,
432 "includeNullMetadata": false,
@@ -470,11 +439,11 @@
439 {
440 "datasource": {
441 "type": "prometheus",
473 - "uid": "${DS_PROMETHEUS}"
442 + "uid": "prometheus"
443 },
444 "disableTextWrap": false,
445 "editorMode": "builder",
477 - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
446 + "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
447 "fullMetaSearch": false,
448 "hide": false,
449 "includeNullMetadata": false,
@@ -487,10 +456,10 @@
456 {
457 "datasource": {
458 "type": "prometheus",
490 - "uid": "${DS_PROMETHEUS}"
459 + "uid": "prometheus"
460 },
461 "editorMode": "code",
493 - "expr": "rate(vllm:time_per_output_token_seconds_sum[$__rate_interval])\n/\nrate(vllm:time_per_output_token_seconds_count[$__rate_interval])",
462 + "expr": "rate(vllm:time_per_output_token_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:time_per_output_token_seconds_count{model_name=\"$model_name\"}[$__rate_interval])",
463 "hide": false,
464 "instant": false,
465 "legendFormat": "Mean",
@@ -504,7 +473,7 @@
473 {
474 "datasource": {
475 "type": "prometheus",
507 - "uid": "${DS_PROMETHEUS}"
476 + "uid": "prometheus"
477 },
478 "description": "Number of requests in RUNNING, WAITING, and SWAPPED state",
479 "fieldConfig": {
@@ -513,7 +482,6 @@
482 "mode": "palette-classic"
483 },
484 "custom": {
516 - "axisBorderShow": false,
485 "axisCenteredZero": false,
486 "axisColorMode": "text",
487 "axisLabel": "",
@@ -527,7 +495,6 @@
495 "tooltip": false,
496 "viz": false
497 },
530 - "insertNulls": false,
498 "lineInterpolation": "linear",
499 "lineWidth": 1,
500 "pointSize": 5,
@@ -585,11 +552,11 @@
552 {
553 "datasource": {
554 "type": "prometheus",
588 - "uid": "${DS_PROMETHEUS}"
555 + "uid": "prometheus"
556 },
557 "disableTextWrap": false,
558 "editorMode": "builder",
592 - "expr": "vllm:num_requests_running",
559 + "expr": "vllm:num_requests_running{model_name=\"$model_name\"}",
560 "fullMetaSearch": false,
561 "includeNullMetadata": true,
562 "instant": false,
@@ -601,11 +568,11 @@
568 {
569 "datasource": {
570 "type": "prometheus",
604 - "uid": "${DS_PROMETHEUS}"
571 + "uid": "prometheus"
572 },
573 "disableTextWrap": false,
574 "editorMode": "builder",
608 - "expr": "vllm:num_requests_swapped",
575 + "expr": "vllm:num_requests_swapped{model_name=\"$model_name\"}",
576 "fullMetaSearch": false,
577 "hide": false,
578 "includeNullMetadata": true,
@@ -618,11 +585,11 @@
585 {
586 "datasource": {
587 "type": "prometheus",
621 - "uid": "${DS_PROMETHEUS}"
588 + "uid": "prometheus"
589 },
590 "disableTextWrap": false,
591 "editorMode": "builder",
625 - "expr": "vllm:num_requests_waiting",
592 + "expr": "vllm:num_requests_waiting{model_name=\"$model_name\"}",
593 "fullMetaSearch": false,
594 "hide": false,
595 "includeNullMetadata": true,
@@ -639,7 +606,7 @@
606 {
607 "datasource": {
608 "type": "prometheus",
642 - "uid": "${DS_PROMETHEUS}"
609 + "uid": "prometheus"
610 },
611 "description": "P50, P90, P95, and P99 TTFT latency in seconds.",
612 "fieldConfig": {
@@ -648,7 +615,6 @@
615 "mode": "palette-classic"
616 },
617 "custom": {
651 - "axisBorderShow": false,
618 "axisCenteredZero": false,
619 "axisColorMode": "text",
620 "axisLabel": "",
@@ -662,7 +628,6 @@
628 "tooltip": false,
629 "viz": false
630 },
665 - "insertNulls": false,
631 "lineInterpolation": "linear",
632 "lineWidth": 1,
633 "pointSize": 5,
@@ -720,11 +685,11 @@
685 {
686 "datasource": {
687 "type": "prometheus",
723 - "uid": "${DS_PROMETHEUS}"
688 + "uid": "prometheus"
689 },
690 "disableTextWrap": false,
691 "editorMode": "builder",
727 - "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
692 + "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
693 "fullMetaSearch": false,
694 "hide": false,
695 "includeNullMetadata": false,
@@ -737,11 +702,11 @@
702 {
703 "datasource": {
704 "type": "prometheus",
740 - "uid": "${DS_PROMETHEUS}"
705 + "uid": "prometheus"
706 },
707 "disableTextWrap": false,
708 "editorMode": "builder",
744 - "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
709 + "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
710 "fullMetaSearch": false,
711 "includeNullMetadata": false,
712 "instant": false,
@@ -753,11 +718,11 @@
718 {
719 "datasource": {
720 "type": "prometheus",
756 - "uid": "${DS_PROMETHEUS}"
721 + "uid": "prometheus"
722 },
723 "disableTextWrap": false,
724 "editorMode": "builder",
760 - "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
725 + "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
726 "fullMetaSearch": false,
727 "hide": false,
728 "includeNullMetadata": false,
@@ -770,11 +735,11 @@
735 {
736 "datasource": {
737 "type": "prometheus",
773 - "uid": "${DS_PROMETHEUS}"
738 + "uid": "prometheus"
739 },
740 "disableTextWrap": false,
741 "editorMode": "builder",
777 - "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
742 + "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket{model_name=\"$model_name\"}[$__rate_interval])))",
743 "fullMetaSearch": false,
744 "hide": false,
745 "includeNullMetadata": false,
@@ -787,10 +752,10 @@
752 {
753 "datasource": {
754 "type": "prometheus",
790 - "uid": "${DS_PROMETHEUS}"
755 + "uid": "prometheus"
756 },
757 "editorMode": "code",
793 - "expr": "rate(vllm:time_to_first_token_seconds_sum[$__rate_interval])\n/\nrate(vllm:time_to_first_token_seconds_count[$__rate_interval])",
758 + "expr": "rate(vllm:time_to_first_token_seconds_sum{model_name=\"$model_name\"}[$__rate_interval])\n/\nrate(vllm:time_to_first_token_seconds_count{model_name=\"$model_name\"}[$__rate_interval])",
759 "hide": false,
760 "instant": false,
761 "legendFormat": "Average",
@@ -804,7 +769,7 @@
769 {
770 "datasource": {
771 "type": "prometheus",
807 - "uid": "${DS_PROMETHEUS}"
772 + "uid": "prometheus"
773 },
774 "description": "Percentage of used cache blocks by vLLM.",
775 "fieldConfig": {
@@ -813,7 +778,6 @@
778 "mode": "palette-classic"
779 },
780 "custom": {
816 - "axisBorderShow": false,
781 "axisCenteredZero": false,
782 "axisColorMode": "text",
783 "axisLabel": "",
@@ -827,7 +791,6 @@
791 "tooltip": false,
792 "viz": false
793 },
830 - "insertNulls": false,
794 "lineInterpolation": "linear",
795 "lineWidth": 1,
796 "pointSize": 5,
@@ -885,10 +848,10 @@
848 {
849 "datasource": {
850 "type": "prometheus",
888 - "uid": "${DS_PROMETHEUS}"
851 + "uid": "prometheus"
852 },
853 "editorMode": "code",
891 - "expr": "vllm:gpu_cache_usage_perc",
854 + "expr": "vllm:gpu_cache_usage_perc{model_name=\"$model_name\"}",
855 "instant": false,
856 "legendFormat": "GPU Cache Usage",
857 "range": true,
@@ -897,10 +860,10 @@
860 {
861 "datasource": {
862 "type": "prometheus",
900 - "uid": "${DS_PROMETHEUS}"
863 + "uid": "prometheus"
864 },
865 "editorMode": "code",
903 - "expr": "vllm:cpu_cache_usage_perc",
866 + "expr": "vllm:cpu_cache_usage_perc{model_name=\"$model_name\"}",
867 "hide": false,
868 "instant": false,
869 "legendFormat": "CPU Cache Usage",
@@ -913,10 +876,39 @@
876 }
877 ],
878 "refresh": "",
916 - "schemaVersion": 39,
879 + "schemaVersion": 37,
880 + "style": "dark",
881 "tags": [],
882 "templating": {
919 - "list": []
883 + "list": [
884 + {
885 + "current": {
886 + "selected": false,
887 + "text": "vllm",
888 + "value": "vllm"
889 + },
890 + "datasource": {
891 + "type": "prometheus",
892 + "uid": "prometheus"
893 + },
894 + "definition": "label_values(model_name)",
895 + "hide": 0,
896 + "includeAll": false,
897 + "label": "model_name",
898 + "multi": false,
899 + "name": "model_name",
900 + "options": [],
901 + "query": {
902 + "query": "label_values(model_name)",
903 + "refId": "StandardVariableQuery"
904 + },
905 + "refresh": 1,
906 + "regex": "",
907 + "skipUrlSync": false,
908 + "sort": 0,
909 + "type": "query"
910 + }
911 + ]
912 },
913 "time": {
914 "from": "now-5m",