@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
yaml 764 lines 22.7 KB
Raw
1 kind: PersesDashboard
2 metadata:
3 name: performance-statistics
4 createdAt: 0001-01-01T00:00:00Z
5 updatedAt: 0001-01-01T00:00:00Z
6 version: 0
7 project: ""
8 spec:
9 display:
10 name: Performance Statistics
11
12 variables:
13 - kind: ListVariable
14 spec:
15 display:
16 name: Deployment_ID
17 hidden: false
18 name: Deployment_id
19 allowAllValue: true
20 allowMultiple: true
21 defaultValue:
22 - $__all
23 sort: alphabetical-asc
24 plugin:
25 kind: PrometheusLabelValuesVariable
26 spec:
27 datasource:
28 kind: PrometheusDatasource
29 name: accelerators-thanos-querier-datasource
30 labelName: model_name
31 matchers:
32 # Any one vllm metric that always carries model_name
33 - vllm:generation_tokens_total{}
34
35 panels:
36 "1":
37 kind: Panel
38 spec:
39 display:
40 name: E2E Latency over Time
41 plugin:
42 kind: TimeSeriesChart
43 spec:
44 legend:
45 mode: table
46 position: bottom
47 queries:
48 - kind: TimeSeriesQuery
49 spec:
50 plugin:
51 kind: PrometheusTimeSeriesQuery
52 spec:
53 datasource:
54 kind: PrometheusDatasource
55 name: accelerators-thanos-querier-datasource
56 # avg latency by model = sum(rate(sum)) / sum(rate(count))
57 query: >
58 sum by (model_name) (rate(vllm:e2e_request_latency_seconds_sum{model_name=~"$Deployment_id"}[$__interval]))
59 /
60 sum by (model_name) (rate(vllm:e2e_request_latency_seconds_count{model_name=~"$Deployment_id"}[$__interval]))
61 seriesNameFormat: '{{model_name}}'
62
63 "2":
64 kind: Panel
65 spec:
66 display:
67 name: E2E Latency (Avg)
68 plugin:
69 kind: StatChart
70 spec:
71 calculation: last-number
72 queries:
73 - kind: TimeSeriesQuery
74 spec:
75 plugin:
76 kind: PrometheusTimeSeriesQuery
77 spec:
78 datasource:
79 kind: PrometheusDatasource
80 name: accelerators-thanos-querier-datasource
81 query: >
82 (sum by (model_name) (increase(vllm:e2e_request_latency_seconds_sum{model_name=~"$Deployment_id"}[$__range])))
83 /
84 (sum by (model_name) (increase(vllm:e2e_request_latency_seconds_count{model_name=~"$Deployment_id"}[$__range])))
85
86 "3":
87 kind: Panel
88 spec:
89 display:
90 name: E2E Latency (P50)
91 plugin:
92 kind: StatChart
93 spec:
94 calculation: last-number
95 queries:
96 - kind: TimeSeriesQuery
97 spec:
98 plugin:
99 kind: PrometheusTimeSeriesQuery
100 spec:
101 datasource:
102 kind: PrometheusDatasource
103 name: accelerators-thanos-querier-datasource
104 query: >
105 histogram_quantile(
106 0.50,
107 sum by (le, model_name) (
108 rate(vllm:e2e_request_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
109 )
110 )
111
112 "4":
113 kind: Panel
114 spec:
115 display:
116 name: E2E Latency (P90)
117 plugin:
118 kind: StatChart
119 spec:
120 calculation: last-number
121 queries:
122 - kind: TimeSeriesQuery
123 spec:
124 plugin:
125 kind: PrometheusTimeSeriesQuery
126 spec:
127 datasource:
128 kind: PrometheusDatasource
129 name: accelerators-thanos-querier-datasource
130 query: >
131 histogram_quantile(
132 0.90,
133 sum by (le, model_name) (
134 rate(vllm:e2e_request_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
135 )
136 )
137
138 "5":
139 kind: Panel
140 spec:
141 display:
142 name: E2E Latency (P99)
143 plugin:
144 kind: StatChart
145 spec:
146 calculation: last-number
147 queries:
148 - kind: TimeSeriesQuery
149 spec:
150 plugin:
151 kind: PrometheusTimeSeriesQuery
152 spec:
153 datasource:
154 kind: PrometheusDatasource
155 name: accelerators-thanos-querier-datasource
156 query: >
157 histogram_quantile(
158 0.99,
159 sum by (le, model_name) (
160 rate(vllm:e2e_request_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
161 )
162 )
163
164 "6":
165 kind: Panel
166 spec:
167 display:
168 name: TTFT over Time
169 plugin:
170 kind: TimeSeriesChart
171 spec:
172 legend:
173 mode: table
174 position: bottom
175 queries:
176 - kind: TimeSeriesQuery
177 spec:
178 plugin:
179 kind: PrometheusTimeSeriesQuery
180 spec:
181 datasource:
182 kind: PrometheusDatasource
183 name: accelerators-thanos-querier-datasource
184 query: >
185 sum by (model_name) (rate(vllm:time_to_first_token_seconds_sum{model_name=~"$Deployment_id"}[$__interval]))
186 /
187 sum by (model_name) (rate(vllm:time_to_first_token_seconds_count{model_name=~"$Deployment_id"}[$__interval]))
188 seriesNameFormat: '{{model_name}}'
189
190 "7":
191 kind: Panel
192 spec:
193 display:
194 name: TTFT (Avg)
195 plugin:
196 kind: StatChart
197 spec:
198 calculation: last-number
199 queries:
200 - kind: TimeSeriesQuery
201 spec:
202 plugin:
203 kind: PrometheusTimeSeriesQuery
204 spec:
205 datasource:
206 kind: PrometheusDatasource
207 name: accelerators-thanos-querier-datasource
208 query: >
209 (sum by (model_name) (increase(vllm:time_to_first_token_seconds_sum{model_name=~"$Deployment_id"}[$__range])))
210 /
211 (sum by (model_name) (increase(vllm:time_to_first_token_seconds_count{model_name=~"$Deployment_id"}[$__range])))
212
213 "8":
214 kind: Panel
215 spec:
216 display:
217 name: TTFT (P50)
218 plugin:
219 kind: StatChart
220 spec:
221 calculation: last-number
222 queries:
223 - kind: TimeSeriesQuery
224 spec:
225 plugin:
226 kind: PrometheusTimeSeriesQuery
227 spec:
228 datasource:
229 kind: PrometheusDatasource
230 name: accelerators-thanos-querier-datasource
231 query: >
232 histogram_quantile(
233 0.50,
234 sum by (le, model_name) (
235 rate(vllm:time_to_first_token_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
236 )
237 )
238
239 "9":
240 kind: Panel
241 spec:
242 display:
243 name: TTFT (P90)
244 plugin:
245 kind: StatChart
246 spec:
247 calculation: last-number
248 queries:
249 - kind: TimeSeriesQuery
250 spec:
251 plugin:
252 kind: PrometheusTimeSeriesQuery
253 spec:
254 datasource:
255 kind: PrometheusDatasource
256 name: accelerators-thanos-querier-datasource
257 query: >
258 histogram_quantile(
259 0.90,
260 sum by (le, model_name) (
261 rate(vllm:time_to_first_token_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
262 )
263 )
264
265 "10":
266 kind: Panel
267 spec:
268 display:
269 name: TTFT (P99)
270 plugin:
271 kind: StatChart
272 spec:
273 calculation: last-number
274 queries:
275 - kind: TimeSeriesQuery
276 spec:
277 plugin:
278 kind: PrometheusTimeSeriesQuery
279 spec:
280 datasource:
281 kind: PrometheusDatasource
282 name: accelerators-thanos-querier-datasource
283 query: >
284 histogram_quantile(
285 0.99,
286 sum by (le, model_name) (
287 rate(vllm:time_to_first_token_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
288 )
289 )
290
291 "11":
292 kind: Panel
293 spec:
294 display:
295 name: ITL (Time per Output Token) over Time
296 plugin:
297 kind: TimeSeriesChart
298 spec:
299 legend:
300 mode: table
301 position: bottom
302 queries:
303 - kind: TimeSeriesQuery
304 spec:
305 plugin:
306 kind: PrometheusTimeSeriesQuery
307 spec:
308 datasource:
309 kind: PrometheusDatasource
310 name: accelerators-thanos-querier-datasource
311 query: >
312 sum by (model_name) (rate(vllm:inter_token_latency_seconds_sum{model_name=~"$Deployment_id"}[$__interval]))
313 /
314 sum by (model_name) (rate(vllm:inter_token_latency_seconds_count{model_name=~"$Deployment_id"}[$__interval]))
315 seriesNameFormat: '{{model_name}}'
316 - kind: TimeSeriesQuery
317 spec:
318 plugin:
319 kind: PrometheusTimeSeriesQuery
320 spec:
321 datasource:
322 kind: PrometheusDatasource
323 name: accelerators-thanos-querier-datasource
324 query: >
325 histogram_quantile(
326 0.50,
327 sum by (le, model_name) (
328 rate(vllm:inter_token_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
329 )
330 )
331 seriesNameFormat: '{{model_name}} p50'
332 - kind: TimeSeriesQuery
333 spec:
334 plugin:
335 kind: PrometheusTimeSeriesQuery
336 spec:
337 datasource:
338 kind: PrometheusDatasource
339 name: accelerators-thanos-querier-datasource
340 query: >
341 histogram_quantile(
342 0.90,
343 sum by (le, model_name) (
344 rate(vllm:inter_token_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
345 )
346 )
347 seriesNameFormat: '{{model_name}} p90'
348 - kind: TimeSeriesQuery
349 spec:
350 plugin:
351 kind: PrometheusTimeSeriesQuery
352 spec:
353 datasource:
354 kind: PrometheusDatasource
355 name: accelerators-thanos-querier-datasource
356 query: >
357 histogram_quantile(
358 0.99,
359 sum by (le, model_name) (
360 rate(vllm:inter_token_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
361 )
362 )
363 seriesNameFormat: '{{model_name}} p99'
364
365 "12":
366 kind: Panel
367 spec:
368 display:
369 name: ITL (Avg)
370 plugin:
371 kind: StatChart
372 spec:
373 calculation: last-number
374 queries:
375 - kind: TimeSeriesQuery
376 spec:
377 plugin:
378 kind: PrometheusTimeSeriesQuery
379 spec:
380 datasource:
381 kind: PrometheusDatasource
382 name: accelerators-thanos-querier-datasource
383 query: >
384 (sum by (model_name) (increase(vllm:inter_token_latency_seconds_sum{model_name=~"$Deployment_id"}[$__range])))
385 /
386 (sum by (model_name) (increase(vllm:inter_token_latency_seconds_count{model_name=~"$Deployment_id"}[$__range])))
387
388 "13":
389 kind: Panel
390 spec:
391 display:
392 name: ITL (P50)
393 plugin:
394 kind: StatChart
395 spec:
396 calculation: last-number
397 queries:
398 - kind: TimeSeriesQuery
399 spec:
400 plugin:
401 kind: PrometheusTimeSeriesQuery
402 spec:
403 datasource:
404 kind: PrometheusDatasource
405 name: accelerators-thanos-querier-datasource
406 query: >
407 histogram_quantile(
408 0.50,
409 sum by (le, model_name) (
410 rate(vllm:inter_token_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
411 )
412 )
413
414 "14":
415 kind: Panel
416 spec:
417 display:
418 name: ITL (P90)
419 plugin:
420 kind: StatChart
421 spec:
422 calculation: last-number
423 queries:
424 - kind: TimeSeriesQuery
425 spec:
426 plugin:
427 kind: PrometheusTimeSeriesQuery
428 spec:
429 datasource:
430 kind: PrometheusDatasource
431 name: accelerators-thanos-querier-datasource
432 query: >
433 histogram_quantile(
434 0.90,
435 sum by (le, model_name) (
436 rate(vllm:inter_token_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
437 )
438 )
439
440 "15":
441 kind: Panel
442 spec:
443 display:
444 name: ITL (P99)
445 plugin:
446 kind: StatChart
447 spec:
448 calculation: last-number
449 queries:
450 - kind: TimeSeriesQuery
451 spec:
452 plugin:
453 kind: PrometheusTimeSeriesQuery
454 spec:
455 datasource:
456 kind: PrometheusDatasource
457 name: accelerators-thanos-querier-datasource
458 query: >
459 histogram_quantile(
460 0.99,
461 sum by (le, model_name) (
462 rate(vllm:inter_token_latency_seconds_bucket{model_name=~"$Deployment_id"}[$__interval])
463 )
464 )
465
466 "16":
467 kind: Panel
468 spec:
469 display:
470 name: TPS (Tokens/sec) over Time
471 plugin:
472 kind: TimeSeriesChart
473 spec:
474 legend:
475 mode: table
476 position: bottom
477 queries:
478 - kind: TimeSeriesQuery
479 spec:
480 plugin:
481 kind: PrometheusTimeSeriesQuery
482 spec:
483 datasource:
484 kind: PrometheusDatasource
485 name: accelerators-thanos-querier-datasource
486 query: >
487 sum by (model_name) (rate(vllm:generation_tokens_total{model_name=~"$Deployment_id"}[$__interval]))
488 seriesNameFormat: '{{model_name}} generation'
489 - kind: TimeSeriesQuery
490 spec:
491 plugin:
492 kind: PrometheusTimeSeriesQuery
493 spec:
494 datasource:
495 kind: PrometheusDatasource
496 name: accelerators-thanos-querier-datasource
497 query: >
498 sum by (model_name) (rate(vllm:prompt_tokens_total{model_name=~"$Deployment_id"}[$__interval]))
499 seriesNameFormat: '{{model_name}} prompt'
500 - kind: TimeSeriesQuery
501 spec:
502 plugin:
503 kind: PrometheusTimeSeriesQuery
504 spec:
505 datasource:
506 kind: PrometheusDatasource
507 name: accelerators-thanos-querier-datasource
508 # overall iteration tokens/sec if exposed
509 query: >
510 rate(vllm:iteration_tokens_total_count[$__interval])
511 seriesNameFormat: 'iteration overall'
512
513 "17":
514 kind: Panel
515 spec:
516 display:
517 name: KV Cache Usage (avg %)
518 plugin:
519 kind: StatChart
520 spec:
521 calculation: last-number
522 queries:
523 - kind: TimeSeriesQuery
524 spec:
525 plugin:
526 kind: PrometheusTimeSeriesQuery
527 spec:
528 datasource:
529 kind: PrometheusDatasource
530 name: accelerators-thanos-querier-datasource
531 # Multiply by 100 so we can read it as a percentage without setting a unit (avoids CUE unit conflicts)
532 query: >
533 100 * avg(vllm:kv_cache_usage_perc)
534
535 "18":
536 kind: Panel
537 spec:
538 display:
539 name: Running Requests by Pod
540 plugin:
541 kind: TimeSeriesChart
542 spec:
543 legend:
544 mode: table
545 position: bottom
546 queries:
547 - kind: TimeSeriesQuery
548 spec:
549 plugin:
550 kind: PrometheusTimeSeriesQuery
551 spec:
552 datasource:
553 kind: PrometheusDatasource
554 name: accelerators-thanos-querier-datasource
555 query: >
556 sum by (pod) (vllm:num_requests_running)
557 seriesNameFormat: '{{pod}}'
558
559 "19":
560 kind: Panel
561 spec:
562 display:
563 name: Waiting Requests by Pod
564 plugin:
565 kind: TimeSeriesChart
566 spec:
567 legend:
568 mode: table
569 position: bottom
570 queries:
571 - kind: TimeSeriesQuery
572 spec:
573 plugin:
574 kind: PrometheusTimeSeriesQuery
575 spec:
576 datasource:
577 kind: PrometheusDatasource
578 name: accelerators-thanos-querier-datasource
579 query: >
580 sum by (pod) (vllm:num_requests_waiting)
581 seriesNameFormat: '{{pod}}'
582
583 "20":
584 kind: Panel
585 spec:
586 display:
587 name: Running Requests (sum)
588 plugin:
589 kind: StatChart
590 spec:
591 calculation: last-number
592 queries:
593 - kind: TimeSeriesQuery
594 spec:
595 plugin:
596 kind: PrometheusTimeSeriesQuery
597 spec:
598 datasource:
599 kind: PrometheusDatasource
600 name: accelerators-thanos-querier-datasource
601 query: sum(vllm:num_requests_running)
602
603 "21":
604 kind: Panel
605 spec:
606 display:
607 name: Waiting Requests (sum)
608 plugin:
609 kind: StatChart
610 spec:
611 calculation: last-number
612 queries:
613 - kind: TimeSeriesQuery
614 spec:
615 plugin:
616 kind: PrometheusTimeSeriesQuery
617 spec:
618 datasource:
619 kind: PrometheusDatasource
620 name: accelerators-thanos-querier-datasource
621 query: sum(vllm:num_requests_waiting)
622
623 layouts:
624 - kind: Grid
625 spec:
626 display:
627 title: Overview
628 items:
629 - x: 0
630 y: 0
631 width: 6
632 height: 3
633 content: { $ref: '#/spec/panels/17' } # KV cache %
634 - x: 6
635 y: 0
636 width: 6
637 height: 3
638 content: { $ref: '#/spec/panels/20' } # running sum
639 - x: 12
640 y: 0
641 width: 6
642 height: 3
643 content: { $ref: '#/spec/panels/21' } # waiting sum
644
645 - kind: Grid
646 spec:
647 display:
648 title: E2E Latency
649 items:
650 - x: 0
651 y: 1
652 width: 10
653 height: 6
654 content: { $ref: '#/spec/panels/1' }
655 - x: 10
656 y: 1
657 width: 7
658 height: 3
659 content: { $ref: '#/spec/panels/2' }
660 - x: 17
661 y: 1
662 width: 7
663 height: 3
664 content: { $ref: '#/spec/panels/3' }
665 - x: 10
666 y: 4
667 width: 7
668 height: 3
669 content: { $ref: '#/spec/panels/4' }
670 - x: 17
671 y: 4
672 width: 7
673 height: 3
674 content: { $ref: '#/spec/panels/5' }
675
676 - kind: Grid
677 spec:
678 display:
679 title: TTFT
680 items:
681 - x: 0
682 y: 8
683 width: 10
684 height: 6
685 content: { $ref: '#/spec/panels/6' }
686 - x: 10
687 y: 8
688 width: 7
689 height: 3
690 content: { $ref: '#/spec/panels/7' }
691 - x: 17
692 y: 8
693 width: 7
694 height: 3
695 content: { $ref: '#/spec/panels/8' }
696 - x: 10
697 y: 11
698 width: 7
699 height: 3
700 content: { $ref: '#/spec/panels/9' }
701 - x: 17
702 y: 11
703 width: 7
704 height: 3
705 content: { $ref: '#/spec/panels/10' }
706
707 - kind: Grid
708 spec:
709 display:
710 title: ITL (Time per Output Token)
711 items:
712 - x: 0
713 y: 15
714 width: 10
715 height: 6
716 content: { $ref: '#/spec/panels/11' }
717 - x: 10
718 y: 15
719 width: 7
720 height: 3
721 content: { $ref: '#/spec/panels/12' }
722 - x: 17
723 y: 15
724 width: 7
725 height: 3
726 content: { $ref: '#/spec/panels/13' }
727 - x: 10
728 y: 18
729 width: 7
730 height: 3
731 content: { $ref: '#/spec/panels/14' }
732 - x: 17
733 y: 18
734 width: 7
735 height: 3
736 content: { $ref: '#/spec/panels/15' }
737
738 - kind: Grid
739 spec:
740 display:
741 title: TPS (Prompt / Generation / Iteration)
742 items:
743 - x: 0
744 y: 22
745 width: 14
746 height: 6
747 content: { $ref: '#/spec/panels/16' }
748
749 - kind: Grid
750 spec:
751 display:
752 title: Per-Pod Request State
753 items:
754 - x: 0
755 y: 28
756 width: 12
757 height: 6
758 content: { $ref: '#/spec/panels/18' }
759 - x: 12
760 y: 28
761 width: 12
762 height: 6
763 content: { $ref: '#/spec/panels/19' }
764