@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 93b38bea5d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Refactor Prometheus and Add Request Level Metrics (#2316)

Robert Shaw committed Jan 31, 2024 at 14:58 UTC 93b38bea5dd03e1b140ca997dfaadef86f8f1855
7 files changed +1234 -102
examples/production_monitoring/README.md new
+54
@@ -0,0 +1,54 @@
1 +# vLLM + Prometheus/Grafana
2 +
3 +This is a simple example that shows you how to connect vLLM metric logging to the Prometheus/Grafana stack. For this example, we launch Prometheus and Grafana via Docker. You can checkout other methods through [Prometheus](https://prometheus.io/) and [Grafana](https://grafana.com/) websites.
4 +
5 +Install:
6 +- [`docker`](https://docs.docker.com/engine/install/)
7 +- [`docker compose`](https://docs.docker.com/compose/install/linux/#install-using-the-repository)
8 +
9 +### Launch
10 +
11 +Prometheus metric logging is enabled by default in the OpenAI-compatible server. Launch via the entrypoint:
12 +```bash
13 +python3 -m vllm.entrypoints.openai.api_server \
14 + --model mistralai/Mistral-7B-v0.1 \
15 + --max-model-len 2048 \
16 + --disable-log-requests
17 +```
18 +
19 +Launch Prometheus and Grafana servers with `docker compose`:
20 +```bash
21 +docker compose up
22 +```
23 +
24 +Submit some sample requests to the server:
25 +```bash
26 +wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
27 +
28 +python3 ../../benchmarks/benchmark_serving.py \
29 + --model mistralai/Mistral-7B-v0.1 \
30 + --tokenizer mistralai/Mistral-7B-v0.1 \
31 + --endpoint /v1/completions \
32 + --dataset ShareGPT_V3_unfiltered_cleaned_split.json \
33 + --request-rate 3.0
34 +```
35 +
36 +Navigating to [`http://localhost:8000/metrics`](http://localhost:8000/metrics) will show the raw Prometheus metrics being exposed by vLLM.
37 +
38 +### Grafana Dashboard
39 +
40 +Navigate to [`http://localhost:3000`](http://localhost:3000). Log in with the default username (`admin`) and password (`admin`).
41 +
42 +#### Add Prometheus Data Source
43 +
44 +Navigate to [`http://localhost:3000/connections/datasources/new`](http://localhost:3000/connections/datasources/new) and select Prometheus.
45 +
46 +On Prometheus configuration page, we need to add the `Prometheus Server URL` in `Connection`. For this setup, Grafana and Prometheus are running in separate containers, but Docker creates DNS name for each containers. You can just use `http://prometheus:9090`.
47 +
48 +Click `Save & Test`. You should get a green check saying "Successfully queried the Prometheus API.".
49 +
50 +#### Import Dashboard
51 +
52 +Navigate to [`http://localhost:3000/dashboard/import`](http://localhost:3000/dashboard/import), upload `grafana.json`, and select the `prometheus` datasource. You should see a screen that looks like the following:
53 +
54 +![Grafana Dashboard Image](https://i.imgur.com/R2vH9VW.png)
examples/production_monitoring/docker-compose.yaml new
+19
@@ -0,0 +1,19 @@
1 +# docker-compose.yaml
2 +version: "3"
3 +
4 +services:
5 + prometheus:
6 + image: prom/prometheus:latest
7 + extra_hosts:
8 + - "host.docker.internal:host-gateway" # allow a direct connection from container to the local machine
9 + ports:
10 + - "9090:9090" # the default port used by Prometheus
11 + volumes:
12 + - ${PWD}/prometheus.yaml:/etc/prometheus/prometheus.yml # mount Prometheus config file
13 +
14 + grafana:
15 + image: grafana/grafana:latest
16 + depends_on:
17 + - prometheus
18 + ports:
19 + - "3000:3000" # the default port used by Grafana
examples/production_monitoring/grafana.json new
+931
@@ -0,0 +1,931 @@
1 +{
2 + "__inputs": [
3 + {
4 + "name": "DS_PROMETHEUS",
5 + "label": "prometheus",
6 + "description": "",
7 + "type": "datasource",
8 + "pluginId": "prometheus",
9 + "pluginName": "Prometheus"
10 + }
11 + ],
12 + "__elements": {},
13 + "__requires": [
14 + {
15 + "type": "grafana",
16 + "id": "grafana",
17 + "name": "Grafana",
18 + "version": "10.2.3"
19 + },
20 + {
21 + "type": "datasource",
22 + "id": "prometheus",
23 + "name": "Prometheus",
24 + "version": "1.0.0"
25 + },
26 + {
27 + "type": "panel",
28 + "id": "timeseries",
29 + "name": "Time series",
30 + "version": ""
31 + }
32 + ],
33 + "annotations": {
34 + "list": [
35 + {
36 + "builtIn": 1,
37 + "datasource": {
38 + "type": "grafana",
39 + "uid": "-- Grafana --"
40 + },
41 + "enable": true,
42 + "hide": true,
43 + "iconColor": "rgba(0, 211, 255, 1)",
44 + "name": "Annotations & Alerts",
45 + "type": "dashboard"
46 + }
47 + ]
48 + },
49 + "description": "Monitoring vLLM Inference Server",
50 + "editable": true,
51 + "fiscalYearStartMonth": 0,
52 + "graphTooltip": 0,
53 + "id": null,
54 + "links": [],
55 + "liveNow": false,
56 + "panels": [
57 + {
58 + "datasource": {
59 + "type": "prometheus",
60 + "uid": "${DS_PROMETHEUS}"
61 + },
62 + "description": "End to end request latency measured in seconds.",
63 + "fieldConfig": {
64 + "defaults": {
65 + "color": {
66 + "mode": "palette-classic"
67 + },
68 + "custom": {
69 + "axisBorderShow": false,
70 + "axisCenteredZero": false,
71 + "axisColorMode": "text",
72 + "axisLabel": "",
73 + "axisPlacement": "auto",
74 + "barAlignment": 0,
75 + "drawStyle": "line",
76 + "fillOpacity": 0,
77 + "gradientMode": "none",
78 + "hideFrom": {
79 + "legend": false,
80 + "tooltip": false,
81 + "viz": false
82 + },
83 + "insertNulls": false,
84 + "lineInterpolation": "linear",
85 + "lineWidth": 1,
86 + "pointSize": 5,
87 + "scaleDistribution": {
88 + "type": "linear"
89 + },
90 + "showPoints": "auto",
91 + "spanNulls": false,
92 + "stacking": {
93 + "group": "A",
94 + "mode": "none"
95 + },
96 + "thresholdsStyle": {
97 + "mode": "off"
98 + }
99 + },
100 + "mappings": [],
101 + "thresholds": {
102 + "mode": "absolute",
103 + "steps": [
104 + {
105 + "color": "green",
106 + "value": null
107 + },
108 + {
109 + "color": "red",
110 + "value": 80
111 + }
112 + ]
113 + },
114 + "unit": "s"
115 + },
116 + "overrides": []
117 + },
118 + "gridPos": {
119 + "h": 8,
120 + "w": 12,
121 + "x": 0,
122 + "y": 0
123 + },
124 + "id": 9,
125 + "options": {
126 + "legend": {
127 + "calcs": [],
128 + "displayMode": "list",
129 + "placement": "bottom",
130 + "showLegend": true
131 + },
132 + "tooltip": {
133 + "mode": "single",
134 + "sort": "none"
135 + }
136 + },
137 + "targets": [
138 + {
139 + "datasource": {
140 + "type": "prometheus",
141 + "uid": "${DS_PROMETHEUS}"
142 + },
143 + "disableTextWrap": false,
144 + "editorMode": "builder",
145 + "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
146 + "fullMetaSearch": false,
147 + "includeNullMetadata": false,
148 + "instant": false,
149 + "legendFormat": "P99",
150 + "range": true,
151 + "refId": "A",
152 + "useBackend": false
153 + },
154 + {
155 + "datasource": {
156 + "type": "prometheus",
157 + "uid": "${DS_PROMETHEUS}"
158 + },
159 + "disableTextWrap": false,
160 + "editorMode": "builder",
161 + "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
162 + "fullMetaSearch": false,
163 + "hide": false,
164 + "includeNullMetadata": false,
165 + "instant": false,
166 + "legendFormat": "P95",
167 + "range": true,
168 + "refId": "B",
169 + "useBackend": false
170 + },
171 + {
172 + "datasource": {
173 + "type": "prometheus",
174 + "uid": "${DS_PROMETHEUS}"
175 + },
176 + "disableTextWrap": false,
177 + "editorMode": "builder",
178 + "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
179 + "fullMetaSearch": false,
180 + "hide": false,
181 + "includeNullMetadata": false,
182 + "instant": false,
183 + "legendFormat": "P90",
184 + "range": true,
185 + "refId": "C",
186 + "useBackend": false
187 + },
188 + {
189 + "datasource": {
190 + "type": "prometheus",
191 + "uid": "${DS_PROMETHEUS}"
192 + },
193 + "disableTextWrap": false,
194 + "editorMode": "builder",
195 + "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:e2e_request_latency_seconds_bucket[$__rate_interval])))",
196 + "fullMetaSearch": false,
197 + "hide": false,
198 + "includeNullMetadata": false,
199 + "instant": false,
200 + "legendFormat": "P50",
201 + "range": true,
202 + "refId": "D",
203 + "useBackend": false
204 + },
205 + {
206 + "datasource": {
207 + "type": "prometheus",
208 + "uid": "${DS_PROMETHEUS}"
209 + },
210 + "editorMode": "code",
211 + "expr": "rate(vllm:e2e_request_latency_seconds_sum[$__rate_interval])\n/\nrate(vllm:e2e_request_latency_seconds_count[$__rate_interval])",
212 + "hide": false,
213 + "instant": false,
214 + "legendFormat": "Average",
215 + "range": true,
216 + "refId": "E"
217 + }
218 + ],
219 + "title": "E2E Request Latency",
220 + "type": "timeseries"
221 + },
222 + {
223 + "datasource": {
224 + "type": "prometheus",
225 + "uid": "${DS_PROMETHEUS}"
226 + },
227 + "description": "Number of tokens processed per second",
228 + "fieldConfig": {
229 + "defaults": {
230 + "color": {
231 + "mode": "palette-classic"
232 + },
233 + "custom": {
234 + "axisBorderShow": false,
235 + "axisCenteredZero": false,
236 + "axisColorMode": "text",
237 + "axisLabel": "",
238 + "axisPlacement": "auto",
239 + "barAlignment": 0,
240 + "drawStyle": "line",
241 + "fillOpacity": 0,
242 + "gradientMode": "none",
243 + "hideFrom": {
244 + "legend": false,
245 + "tooltip": false,
246 + "viz": false
247 + },
248 + "insertNulls": false,
249 + "lineInterpolation": "linear",
250 + "lineWidth": 1,
251 + "pointSize": 5,
252 + "scaleDistribution": {
253 + "type": "linear"
254 + },
255 + "showPoints": "auto",
256 + "spanNulls": false,
257 + "stacking": {
258 + "group": "A",
259 + "mode": "none"
260 + },
261 + "thresholdsStyle": {
262 + "mode": "off"
263 + }
264 + },
265 + "mappings": [],
266 + "thresholds": {
267 + "mode": "absolute",
268 + "steps": [
269 + {
270 + "color": "green",
271 + "value": null
272 + },
273 + {
274 + "color": "red",
275 + "value": 80
276 + }
277 + ]
278 + }
279 + },
280 + "overrides": []
281 + },
282 + "gridPos": {
283 + "h": 8,
284 + "w": 12,
285 + "x": 12,
286 + "y": 0
287 + },
288 + "id": 8,
289 + "options": {
290 + "legend": {
291 + "calcs": [],
292 + "displayMode": "list",
293 + "placement": "bottom",
294 + "showLegend": true
295 + },
296 + "tooltip": {
297 + "mode": "single",
298 + "sort": "none"
299 + }
300 + },
301 + "targets": [
302 + {
303 + "datasource": {
304 + "type": "prometheus",
305 + "uid": "${DS_PROMETHEUS}"
306 + },
307 + "disableTextWrap": false,
308 + "editorMode": "builder",
309 + "expr": "rate(vllm:prompt_tokens_total[$__rate_interval])",
310 + "fullMetaSearch": false,
311 + "includeNullMetadata": false,
312 + "instant": false,
313 + "legendFormat": "Prompt Tokens/Sec",
314 + "range": true,
315 + "refId": "A",
316 + "useBackend": false
317 + },
318 + {
319 + "datasource": {
320 + "type": "prometheus",
321 + "uid": "${DS_PROMETHEUS}"
322 + },
323 + "disableTextWrap": false,
324 + "editorMode": "builder",
325 + "expr": "rate(vllm:generation_tokens_total[$__rate_interval])",
326 + "fullMetaSearch": false,
327 + "hide": false,
328 + "includeNullMetadata": false,
329 + "instant": false,
330 + "legendFormat": "Generation Tokens/Sec",
331 + "range": true,
332 + "refId": "B",
333 + "useBackend": false
334 + }
335 + ],
336 + "title": "Token Throughput",
337 + "type": "timeseries"
338 + },
339 + {
340 + "datasource": {
341 + "type": "prometheus",
342 + "uid": "${DS_PROMETHEUS}"
343 + },
344 + "description": "Inter token latency in seconds.",
345 + "fieldConfig": {
346 + "defaults": {
347 + "color": {
348 + "mode": "palette-classic"
349 + },
350 + "custom": {
351 + "axisBorderShow": false,
352 + "axisCenteredZero": false,
353 + "axisColorMode": "text",
354 + "axisLabel": "",
355 + "axisPlacement": "auto",
356 + "barAlignment": 0,
357 + "drawStyle": "line",
358 + "fillOpacity": 0,
359 + "gradientMode": "none",
360 + "hideFrom": {
361 + "legend": false,
362 + "tooltip": false,
363 + "viz": false
364 + },
365 + "insertNulls": false,
366 + "lineInterpolation": "linear",
367 + "lineWidth": 1,
368 + "pointSize": 5,
369 + "scaleDistribution": {
370 + "type": "linear"
371 + },
372 + "showPoints": "auto",
373 + "spanNulls": false,
374 + "stacking": {
375 + "group": "A",
376 + "mode": "none"
377 + },
378 + "thresholdsStyle": {
379 + "mode": "off"
380 + }
381 + },
382 + "mappings": [],
383 + "thresholds": {
384 + "mode": "absolute",
385 + "steps": [
386 + {
387 + "color": "green",
388 + "value": null
389 + },
390 + {
391 + "color": "red",
392 + "value": 80
393 + }
394 + ]
395 + },
396 + "unit": "s"
397 + },
398 + "overrides": []
399 + },
400 + "gridPos": {
401 + "h": 8,
402 + "w": 12,
403 + "x": 0,
404 + "y": 8
405 + },
406 + "id": 10,
407 + "options": {
408 + "legend": {
409 + "calcs": [],
410 + "displayMode": "list",
411 + "placement": "bottom",
412 + "showLegend": true
413 + },
414 + "tooltip": {
415 + "mode": "single",
416 + "sort": "none"
417 + }
418 + },
419 + "targets": [
420 + {
421 + "datasource": {
422 + "type": "prometheus",
423 + "uid": "${DS_PROMETHEUS}"
424 + },
425 + "disableTextWrap": false,
426 + "editorMode": "builder",
427 + "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
428 + "fullMetaSearch": false,
429 + "includeNullMetadata": false,
430 + "instant": false,
431 + "legendFormat": "P99",
432 + "range": true,
433 + "refId": "A",
434 + "useBackend": false
435 + },
436 + {
437 + "datasource": {
438 + "type": "prometheus",
439 + "uid": "${DS_PROMETHEUS}"
440 + },
441 + "disableTextWrap": false,
442 + "editorMode": "builder",
443 + "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
444 + "fullMetaSearch": false,
445 + "hide": false,
446 + "includeNullMetadata": false,
447 + "instant": false,
448 + "legendFormat": "P95",
449 + "range": true,
450 + "refId": "B",
451 + "useBackend": false
452 + },
453 + {
454 + "datasource": {
455 + "type": "prometheus",
456 + "uid": "${DS_PROMETHEUS}"
457 + },
458 + "disableTextWrap": false,
459 + "editorMode": "builder",
460 + "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
461 + "fullMetaSearch": false,
462 + "hide": false,
463 + "includeNullMetadata": false,
464 + "instant": false,
465 + "legendFormat": "P90",
466 + "range": true,
467 + "refId": "C",
468 + "useBackend": false
469 + },
470 + {
471 + "datasource": {
472 + "type": "prometheus",
473 + "uid": "${DS_PROMETHEUS}"
474 + },
475 + "disableTextWrap": false,
476 + "editorMode": "builder",
477 + "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_per_output_token_seconds_bucket[$__rate_interval])))",
478 + "fullMetaSearch": false,
479 + "hide": false,
480 + "includeNullMetadata": false,
481 + "instant": false,
482 + "legendFormat": "P50",
483 + "range": true,
484 + "refId": "D",
485 + "useBackend": false
486 + },
487 + {
488 + "datasource": {
489 + "type": "prometheus",
490 + "uid": "${DS_PROMETHEUS}"
491 + },
492 + "editorMode": "code",
493 + "expr": "rate(vllm:time_per_output_token_seconds_sum[$__rate_interval])\n/\nrate(vllm:time_per_output_token_seconds_count[$__rate_interval])",
494 + "hide": false,
495 + "instant": false,
496 + "legendFormat": "Mean",
497 + "range": true,
498 + "refId": "E"
499 + }
500 + ],
501 + "title": "Time Per Output Token Latency",
502 + "type": "timeseries"
503 + },
504 + {
505 + "datasource": {
506 + "type": "prometheus",
507 + "uid": "${DS_PROMETHEUS}"
508 + },
509 + "description": "Number of requests in RUNNING, WAITING, and SWAPPED state",
510 + "fieldConfig": {
511 + "defaults": {
512 + "color": {
513 + "mode": "palette-classic"
514 + },
515 + "custom": {
516 + "axisBorderShow": false,
517 + "axisCenteredZero": false,
518 + "axisColorMode": "text",
519 + "axisLabel": "",
520 + "axisPlacement": "auto",
521 + "barAlignment": 0,
522 + "drawStyle": "line",
523 + "fillOpacity": 0,
524 + "gradientMode": "none",
525 + "hideFrom": {
526 + "legend": false,
527 + "tooltip": false,
528 + "viz": false
529 + },
530 + "insertNulls": false,
531 + "lineInterpolation": "linear",
532 + "lineWidth": 1,
533 + "pointSize": 5,
534 + "scaleDistribution": {
535 + "type": "linear"
536 + },
537 + "showPoints": "auto",
538 + "spanNulls": false,
539 + "stacking": {
540 + "group": "A",
541 + "mode": "none"
542 + },
543 + "thresholdsStyle": {
544 + "mode": "off"
545 + }
546 + },
547 + "mappings": [],
548 + "thresholds": {
549 + "mode": "absolute",
550 + "steps": [
551 + {
552 + "color": "green",
553 + "value": null
554 + },
555 + {
556 + "color": "red",
557 + "value": 80
558 + }
559 + ]
560 + },
561 + "unit": "none"
562 + },
563 + "overrides": []
564 + },
565 + "gridPos": {
566 + "h": 8,
567 + "w": 12,
568 + "x": 12,
569 + "y": 8
570 + },
571 + "id": 3,
572 + "options": {
573 + "legend": {
574 + "calcs": [],
575 + "displayMode": "list",
576 + "placement": "bottom",
577 + "showLegend": true
578 + },
579 + "tooltip": {
580 + "mode": "single",
581 + "sort": "none"
582 + }
583 + },
584 + "targets": [
585 + {
586 + "datasource": {
587 + "type": "prometheus",
588 + "uid": "${DS_PROMETHEUS}"
589 + },
590 + "disableTextWrap": false,
591 + "editorMode": "builder",
592 + "expr": "vllm:num_requests_running",
593 + "fullMetaSearch": false,
594 + "includeNullMetadata": true,
595 + "instant": false,
596 + "legendFormat": "Num Running",
597 + "range": true,
598 + "refId": "A",
599 + "useBackend": false
600 + },
601 + {
602 + "datasource": {
603 + "type": "prometheus",
604 + "uid": "${DS_PROMETHEUS}"
605 + },
606 + "disableTextWrap": false,
607 + "editorMode": "builder",
608 + "expr": "vllm:num_requests_swapped",
609 + "fullMetaSearch": false,
610 + "hide": false,
611 + "includeNullMetadata": true,
612 + "instant": false,
613 + "legendFormat": "Num Swapped",
614 + "range": true,
615 + "refId": "B",
616 + "useBackend": false
617 + },
618 + {
619 + "datasource": {
620 + "type": "prometheus",
621 + "uid": "${DS_PROMETHEUS}"
622 + },
623 + "disableTextWrap": false,
624 + "editorMode": "builder",
625 + "expr": "vllm:num_requests_waiting",
626 + "fullMetaSearch": false,
627 + "hide": false,
628 + "includeNullMetadata": true,
629 + "instant": false,
630 + "legendFormat": "Num Waiting",
631 + "range": true,
632 + "refId": "C",
633 + "useBackend": false
634 + }
635 + ],
636 + "title": "Scheduler State",
637 + "type": "timeseries"
638 + },
639 + {
640 + "datasource": {
641 + "type": "prometheus",
642 + "uid": "${DS_PROMETHEUS}"
643 + },
644 + "description": "P50, P90, P95, and P99 TTFT latency in seconds.",
645 + "fieldConfig": {
646 + "defaults": {
647 + "color": {
648 + "mode": "palette-classic"
649 + },
650 + "custom": {
651 + "axisBorderShow": false,
652 + "axisCenteredZero": false,
653 + "axisColorMode": "text",
654 + "axisLabel": "",
655 + "axisPlacement": "auto",
656 + "barAlignment": 0,
657 + "drawStyle": "line",
658 + "fillOpacity": 0,
659 + "gradientMode": "none",
660 + "hideFrom": {
661 + "legend": false,
662 + "tooltip": false,
663 + "viz": false
664 + },
665 + "insertNulls": false,
666 + "lineInterpolation": "linear",
667 + "lineWidth": 1,
668 + "pointSize": 5,
669 + "scaleDistribution": {
670 + "type": "linear"
671 + },
672 + "showPoints": "auto",
673 + "spanNulls": false,
674 + "stacking": {
675 + "group": "A",
676 + "mode": "none"
677 + },
678 + "thresholdsStyle": {
679 + "mode": "off"
680 + }
681 + },
682 + "mappings": [],
683 + "thresholds": {
684 + "mode": "absolute",
685 + "steps": [
686 + {
687 + "color": "green",
688 + "value": null
689 + },
690 + {
691 + "color": "red",
692 + "value": 80
693 + }
694 + ]
695 + },
696 + "unit": "s"
697 + },
698 + "overrides": []
699 + },
700 + "gridPos": {
701 + "h": 8,
702 + "w": 12,
703 + "x": 0,
704 + "y": 16
705 + },
706 + "id": 5,
707 + "options": {
708 + "legend": {
709 + "calcs": [],
710 + "displayMode": "list",
711 + "placement": "bottom",
712 + "showLegend": true
713 + },
714 + "tooltip": {
715 + "mode": "single",
716 + "sort": "none"
717 + }
718 + },
719 + "targets": [
720 + {
721 + "datasource": {
722 + "type": "prometheus",
723 + "uid": "${DS_PROMETHEUS}"
724 + },
725 + "disableTextWrap": false,
726 + "editorMode": "builder",
727 + "expr": "histogram_quantile(0.99, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
728 + "fullMetaSearch": false,
729 + "hide": false,
730 + "includeNullMetadata": false,
731 + "instant": false,
732 + "legendFormat": "P99",
733 + "range": true,
734 + "refId": "A",
735 + "useBackend": false
736 + },
737 + {
738 + "datasource": {
739 + "type": "prometheus",
740 + "uid": "${DS_PROMETHEUS}"
741 + },
742 + "disableTextWrap": false,
743 + "editorMode": "builder",
744 + "expr": "histogram_quantile(0.95, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
745 + "fullMetaSearch": false,
746 + "includeNullMetadata": false,
747 + "instant": false,
748 + "legendFormat": "P95",
749 + "range": true,
750 + "refId": "B",
751 + "useBackend": false
752 + },
753 + {
754 + "datasource": {
755 + "type": "prometheus",
756 + "uid": "${DS_PROMETHEUS}"
757 + },
758 + "disableTextWrap": false,
759 + "editorMode": "builder",
760 + "expr": "histogram_quantile(0.9, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
761 + "fullMetaSearch": false,
762 + "hide": false,
763 + "includeNullMetadata": false,
764 + "instant": false,
765 + "legendFormat": "P90",
766 + "range": true,
767 + "refId": "C",
768 + "useBackend": false
769 + },
770 + {
771 + "datasource": {
772 + "type": "prometheus",
773 + "uid": "${DS_PROMETHEUS}"
774 + },
775 + "disableTextWrap": false,
776 + "editorMode": "builder",
777 + "expr": "histogram_quantile(0.5, sum by(le) (rate(vllm:time_to_first_token_seconds_bucket[$__rate_interval])))",
778 + "fullMetaSearch": false,
779 + "hide": false,
780 + "includeNullMetadata": false,
781 + "instant": false,
782 + "legendFormat": "P50",
783 + "range": true,
784 + "refId": "D",
785 + "useBackend": false
786 + },
787 + {
788 + "datasource": {
789 + "type": "prometheus",
790 + "uid": "${DS_PROMETHEUS}"
791 + },
792 + "editorMode": "code",
793 + "expr": "rate(vllm:time_to_first_token_seconds_sum[$__rate_interval])\n/\nrate(vllm:time_to_first_token_seconds_count[$__rate_interval])",
794 + "hide": false,
795 + "instant": false,
796 + "legendFormat": "Average",
797 + "range": true,
798 + "refId": "E"
799 + }
800 + ],
801 + "title": "Time To First Token Latency",
802 + "type": "timeseries"
803 + },
804 + {
805 + "datasource": {
806 + "type": "prometheus",
807 + "uid": "${DS_PROMETHEUS}"
808 + },
809 + "description": "Percentage of used cache blocks by vLLM.",
810 + "fieldConfig": {
811 + "defaults": {
812 + "color": {
813 + "mode": "palette-classic"
814 + },
815 + "custom": {
816 + "axisBorderShow": false,
817 + "axisCenteredZero": false,
818 + "axisColorMode": "text",
819 + "axisLabel": "",
820 + "axisPlacement": "auto",
821 + "barAlignment": 0,
822 + "drawStyle": "line",
823 + "fillOpacity": 0,
824 + "gradientMode": "none",
825 + "hideFrom": {
826 + "legend": false,
827 + "tooltip": false,
828 + "viz": false
829 + },
830 + "insertNulls": false,
831 + "lineInterpolation": "linear",
832 + "lineWidth": 1,
833 + "pointSize": 5,
834 + "scaleDistribution": {
835 + "type": "linear"
836 + },
837 + "showPoints": "auto",
838 + "spanNulls": false,
839 + "stacking": {
840 + "group": "A",
841 + "mode": "none"
842 + },
843 + "thresholdsStyle": {
844 + "mode": "off"
845 + }
846 + },
847 + "mappings": [],
848 + "thresholds": {
849 + "mode": "absolute",
850 + "steps": [
851 + {
852 + "color": "green",
853 + "value": null
854 + },
855 + {
856 + "color": "red",
857 + "value": 80
858 + }
859 + ]
860 + },
861 + "unit": "percentunit"
862 + },
863 + "overrides": []
864 + },
865 + "gridPos": {
866 + "h": 8,
867 + "w": 12,
868 + "x": 12,
869 + "y": 16
870 + },
871 + "id": 4,
872 + "options": {
873 + "legend": {
874 + "calcs": [],
875 + "displayMode": "list",
876 + "placement": "bottom",
877 + "showLegend": true
878 + },
879 + "tooltip": {
880 + "mode": "single",
881 + "sort": "none"
882 + }
883 + },
884 + "targets": [
885 + {
886 + "datasource": {
887 + "type": "prometheus",
888 + "uid": "${DS_PROMETHEUS}"
889 + },
890 + "editorMode": "code",
891 + "expr": "vllm:gpu_cache_usage_perc",
892 + "instant": false,
893 + "legendFormat": "GPU Cache Usage",
894 + "range": true,
895 + "refId": "A"
896 + },
897 + {
898 + "datasource": {
899 + "type": "prometheus",
900 + "uid": "${DS_PROMETHEUS}"
901 + },
902 + "editorMode": "code",
903 + "expr": "vllm:cpu_cache_usage_perc",
904 + "hide": false,
905 + "instant": false,
906 + "legendFormat": "CPU Cache Usage",
907 + "range": true,
908 + "refId": "B"
909 + }
910 + ],
911 + "title": "Cache Utilization",
912 + "type": "timeseries"
913 + }
914 + ],
915 + "refresh": "",
916 + "schemaVersion": 39,
917 + "tags": [],
918 + "templating": {
919 + "list": []
920 + },
921 + "time": {
922 + "from": "now-5m",
923 + "to": "now"
924 + },
925 + "timepicker": {},
926 + "timezone": "",
927 + "title": "vLLM",
928 + "uid": "b281712d-8bff-41ef-9f3f-71ad43c05e9b",
929 + "version": 2,
930 + "weekStart": ""
931 +}
examples/production_monitoring/prometheus.yaml new
+10
@@ -0,0 +1,10 @@
1 +# prometheus.yaml
2 +global:
3 + scrape_interval: 5s
4 + evaluation_interval: 30s
5 +
6 +scrape_configs:
7 + - job_name: vllm
8 + static_configs:
9 + - targets:
10 + - 'host.docker.internal:8000'
vllm/engine/llm_engine.py
+69 -79
@@ -10,7 +10,7 @@ from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
10 SchedulerConfig, LoRAConfig)
11 from vllm.core.scheduler import Scheduler, SchedulerOutputs
12 from vllm.engine.arg_utils import EngineArgs
13 -from vllm.engine.metrics import record_metrics
13 +from vllm.engine.metrics import StatLogger, Stats
14 from vllm.engine.ray_utils import RayWorkerVllm, initialize_cluster, ray
15 from vllm.logger import init_logger
16 from vllm.outputs import RequestOutput
@@ -28,8 +28,7 @@ if TYPE_CHECKING:
28 from ray.util.placement_group import PlacementGroup
29
30 logger = init_logger(__name__)
31 -
32 -_LOGGING_INTERVAL_SEC = 5
31 +_LOCAL_LOGGING_INTERVAL_SEC = 5
32
33
34 class LLMEngine:
@@ -116,12 +115,10 @@ class LLMEngine:
115 # Create the scheduler.
116 self.scheduler = Scheduler(scheduler_config, cache_config, lora_config)
117
119 - # Logging.
120 - self.last_logging_time = 0.0
121 - # List of (timestamp, num_tokens)
122 - self.num_prompt_tokens: List[Tuple[float, int]] = []
123 - # List of (timestamp, num_tokens)
124 - self.num_generation_tokens: List[Tuple[float, int]] = []
118 + # Metric Logging.
119 + if self.log_stats:
120 + self.stat_logger = StatLogger(
121 + local_interval=_LOCAL_LOGGING_INTERVAL_SEC)
122
123 def get_tokenizer_for_seq(self, sequence: Sequence):
124 return self.tokenizer.get_lora_tokenizer(sequence.lora_request)
@@ -537,6 +534,7 @@ class LLMEngine:
534
535 def _process_sequence_group_outputs(self, seq_group: SequenceGroup,
536 outputs: SequenceGroupOutput) -> None:
537 +
538 # Process prompt logprobs
539 prompt_logprobs = outputs.prompt_logprobs
540 if prompt_logprobs is not None:
@@ -732,10 +730,10 @@ class LLMEngine:
730 and not seq_group.prefix.computed):
731 seq_group.prefix.computed = True
732
733 + # Log stats.
734 if self.log_stats:
736 - # Log the system stats.
737 - self._log_system_stats(scheduler_outputs.prompt_run,
738 - scheduler_outputs.num_batched_tokens)
735 + self.stat_logger.log(self._get_stats(scheduler_outputs))
736 +
737 return request_outputs
738
739 def step(self) -> List[RequestOutput]:
@@ -810,81 +808,73 @@ class LLMEngine:
808 return self._process_model_outputs(output, scheduler_outputs)
809
810 def do_log_stats(self) -> None:
813 - self._log_system_stats(False, 0)
811 + """Forced log when no requests active."""
812 + if self.log_stats:
813 + self.stat_logger.log(self._get_stats(scheduler_outputs=None))
814
815 - def _log_system_stats(
816 - self,
817 - prompt_run: bool,
818 - num_batched_tokens: int,
819 - ) -> None:
815 + def _get_stats(self,
816 + scheduler_outputs: Optional[SchedulerOutputs]) -> Stats:
817 + """Get Stats to be Logged to Prometheus."""
818 now = time.monotonic()
821 - # Log the number of batched input tokens.
822 - if prompt_run:
823 - self.num_prompt_tokens.append((now, num_batched_tokens))
824 - else:
825 - self.num_generation_tokens.append((now, num_batched_tokens))
819
827 - should_log = now - self.last_logging_time >= _LOGGING_INTERVAL_SEC
828 - if not should_log:
829 - return
820 + # KV Cache Usage in %.
821 + num_total_gpu = self.cache_config.num_gpu_blocks
822 + num_free_gpu = self.scheduler.block_manager.get_num_free_gpu_blocks()
823 + gpu_cache_usage = 1.0 - (num_free_gpu / num_total_gpu)
824
831 - # Discard the old stats.
832 - self.num_prompt_tokens = [(t, n) for t, n in self.num_prompt_tokens
833 - if now - t < _LOGGING_INTERVAL_SEC]
834 - self.num_generation_tokens = [(t, n)
835 - for t, n in self.num_generation_tokens
836 - if now - t < _LOGGING_INTERVAL_SEC]
837 -
838 - if len(self.num_prompt_tokens) > 1:
839 - total_num_tokens = sum(n for _, n in self.num_prompt_tokens[:-1])
840 - window = now - self.num_prompt_tokens[0][0]
841 - avg_prompt_throughput = total_num_tokens / window
842 - else:
843 - avg_prompt_throughput = 0.0
844 - if len(self.num_generation_tokens) > 1:
845 - total_num_tokens = sum(n
846 - for _, n in self.num_generation_tokens[:-1])
847 - window = now - self.num_generation_tokens[0][0]
848 - avg_generation_throughput = total_num_tokens / window
849 - else:
850 - avg_generation_throughput = 0.0
851 -
852 - total_num_gpu_blocks = self.cache_config.num_gpu_blocks
853 - num_free_gpu_blocks = (
854 - self.scheduler.block_manager.get_num_free_gpu_blocks())
855 - num_used_gpu_blocks = total_num_gpu_blocks - num_free_gpu_blocks
856 - gpu_cache_usage = num_used_gpu_blocks / total_num_gpu_blocks
857 -
858 - total_num_cpu_blocks = self.cache_config.num_cpu_blocks
859 - if total_num_cpu_blocks > 0:
860 - num_free_cpu_blocks = (
861 - self.scheduler.block_manager.get_num_free_cpu_blocks())
862 - num_used_cpu_blocks = total_num_cpu_blocks - num_free_cpu_blocks
863 - cpu_cache_usage = num_used_cpu_blocks / total_num_cpu_blocks
864 - else:
865 - cpu_cache_usage = 0.0
866 -
867 - record_metrics(
868 - avg_prompt_throughput=avg_prompt_throughput,
869 - avg_generation_throughput=avg_generation_throughput,
870 - scheduler_running=len(self.scheduler.running),
871 - scheduler_swapped=len(self.scheduler.swapped),
872 - scheduler_waiting=len(self.scheduler.waiting),
825 + num_total_cpu = self.cache_config.num_cpu_blocks
826 + cpu_cache_usage = 0.
827 + if num_total_cpu > 0:
828 + num_free_cpu = self.scheduler.block_manager.get_num_free_cpu_blocks(
829 + )
830 + cpu_cache_usage = 1.0 - (num_free_cpu / num_total_cpu)
831 +
832 + # Scheduler State
833 + num_running = len(self.scheduler.running)
834 + num_swapped = len(self.scheduler.swapped)
835 + num_waiting = len(self.scheduler.waiting)
836 +
837 + # Iteration stats if we have scheduler output.
838 + num_prompt_tokens = 0
839 + num_generation_tokens = 0
840 + time_to_first_tokens = []
841 + time_per_output_tokens = []
842 + time_e2e_requests = []
843 + if scheduler_outputs is not None:
844 + prompt_run = scheduler_outputs.prompt_run
845 +
846 + # Number of Tokens.
847 + if prompt_run:
848 + num_prompt_tokens = scheduler_outputs.num_batched_tokens
849 + else:
850 + num_generation_tokens = scheduler_outputs.num_batched_tokens
851 +
852 + # Latency Timings.
853 + time_last_iters = []
854 + for seq_group in scheduler_outputs.scheduled_seq_groups:
855 + # Time since last token. (n.b. updates seq_group.last_token_time)
856 + time_last_iters.append(seq_group.get_last_latency(now))
857 + # Time since arrival for all finished requests.
858 + if seq_group.is_finished():
859 + time_e2e_requests.append(now - seq_group.arrival_time)
860 +
861 + time_to_first_tokens = time_last_iters if prompt_run else []
862 + time_per_output_tokens = [] if prompt_run else time_last_iters
863 +
864 + return Stats(
865 + now=now,
866 + num_running=num_running,
867 + num_swapped=num_swapped,
868 + num_waiting=num_waiting,
869 gpu_cache_usage=gpu_cache_usage,
870 cpu_cache_usage=cpu_cache_usage,
871 + num_prompt_tokens=num_prompt_tokens,
872 + num_generation_tokens=num_generation_tokens,
873 + time_to_first_tokens=time_to_first_tokens,
874 + time_per_output_tokens=time_per_output_tokens,
875 + time_e2e_requests=time_e2e_requests,
876 )
877
877 - logger.info("Avg prompt throughput: "
878 - f"{avg_prompt_throughput:.1f} tokens/s, "
879 - "Avg generation throughput: "
880 - f"{avg_generation_throughput:.1f} tokens/s, "
881 - f"Running: {len(self.scheduler.running)} reqs, "
882 - f"Swapped: {len(self.scheduler.swapped)} reqs, "
883 - f"Pending: {len(self.scheduler.waiting)} reqs, "
884 - f"GPU KV cache usage: {gpu_cache_usage * 100:.1f}%, "
885 - f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
886 - self.last_logging_time = now
887 -
878 def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
879 """Decodes the new token for a sequence."""
880 (new_tokens, new_output_text, prefix_offset,
vllm/engine/metrics.py
+144 -22
@@ -1,4 +1,19 @@
1 -from aioprometheus import Gauge
1 +from vllm.logger import init_logger
2 +from aioprometheus import Counter, Gauge, Histogram
3 +
4 +import time
5 +import numpy as np
6 +from typing import List
7 +from dataclasses import dataclass
8 +
9 +logger = init_logger(__name__)
10 +
11 +labels = {}
12 +
13 +
14 +def add_global_metrics_labels(**kwargs):
15 + labels.update(kwargs)
16 +
17
18 # The begin-* and end* here are used by the documentation generator
19 # to extract the metrics definitions.
@@ -9,12 +24,16 @@ gauge_avg_prompt_throughput = Gauge("vllm:avg_prompt_throughput_toks_per_s",
24 gauge_avg_generation_throughput = Gauge(
25 "vllm:avg_generation_throughput_toks_per_s",
26 "Average generation throughput in tokens/s.")
27 +counter_prompt_tokens = Counter("vllm:prompt_tokens_total",
28 + "Number of prefill tokens processed.")
29 +counter_generation_tokens = Counter("vllm:generation_tokens_total",
30 + "Number of generation tokens processed.")
31
32 gauge_scheduler_running = Gauge(
33 "vllm:num_requests_running",
15 - "Number of requests that is currently running for inference.")
34 + "Number of requests currently running on GPU.")
35 gauge_scheduler_swapped = Gauge("vllm:num_requests_swapped",
17 - "Number requests swapped to CPU.")
36 + "Number of requests swapped to CPU.")
37 gauge_scheduler_waiting = Gauge("vllm:num_requests_waiting",
38 "Number of requests waiting to be processed.")
39
@@ -24,28 +43,131 @@ gauge_gpu_cache_usage = Gauge(
43 gauge_cpu_cache_usage = Gauge(
44 "vllm:cpu_cache_usage_perc",
45 "CPU KV-cache usage. 1 means 100 percent usage.")
46 +
47 +histogram_time_to_first_token = Histogram(
48 + "vllm:time_to_first_token_seconds",
49 + "Histogram of time to first token in seconds.",
50 + buckets=[
51 + 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5, 0.75, 1.0,
52 + 2.5, 5.0, 7.5, 10.0
53 + ])
54 +histogram_time_per_output_tokens = Histogram(
55 + "vllm:time_per_output_token_seconds",
56 + "Histogram of time per output token in seconds.",
57 + buckets=[
58 + 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 2.5
59 + ])
60 +histogram_e2e_request_latency = Histogram(
61 + "vllm:e2e_request_latency_seconds",
62 + "Histogram of end to end request latency in seconds.",
63 + buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
64 # end-metrics-definitions
65
29 -labels = {}
66
67 +@dataclass
68 +class Stats:
69 + """Created by LLMEngine for use by StatLogger."""
70 + now: float
71
32 -def add_global_metrics_labels(**kwargs):
33 - labels.update(kwargs)
72 + # System stats.
73 + num_running: int
74 + num_waiting: int
75 + num_swapped: int
76 + gpu_cache_usage: float
77 + cpu_cache_usage: float
78 +
79 + # Raw stats from last model iteration.
80 + num_prompt_tokens: int
81 + num_generation_tokens: int
82 + time_to_first_tokens: List[float]
83 + time_per_output_tokens: List[float]
84 + time_e2e_requests: List[float]
85 +
86 +
87 +class StatLogger:
88 + """StatLogger is used LLMEngine to log to Promethus and Stdout."""
89 +
90 + def __init__(self, local_interval: float) -> None:
91 + # Metadata for logging locally.
92 + self.last_local_log = time.monotonic()
93 + self.local_interval = local_interval
94 +
95 + # Tracked stats over current local logging interval.
96 + self.num_prompt_tokens: List[int] = []
97 + self.num_generation_tokens: List[int] = []
98 +
99 + def _get_throughput(self, tracked_stats: List[int], now: float) -> float:
100 + return float(np.sum(tracked_stats) / (now - self.last_local_log))
101 +
102 + def _local_interval_elapsed(self, now: float) -> bool:
103 + elapsed_time = now - self.last_local_log
104 + return elapsed_time > self.local_interval
105 +
106 + def _log_prometheus(self, stats: Stats) -> None:
107 + # Set system stat gauges.
108 + gauge_scheduler_running.set(labels, stats.num_running)
109 + gauge_scheduler_swapped.set(labels, stats.num_swapped)
110 + gauge_scheduler_waiting.set(labels, stats.num_waiting)
111 + gauge_gpu_cache_usage.set(labels, stats.gpu_cache_usage)
112 + gauge_cpu_cache_usage.set(labels, stats.cpu_cache_usage)
113 +
114 + # Add to token counters.
115 + counter_prompt_tokens.add(labels, stats.num_prompt_tokens)
116 + counter_generation_tokens.add(labels, stats.num_generation_tokens)
117 +
118 + # Observe request level latencies in histograms.
119 + for ttft in stats.time_to_first_tokens:
120 + histogram_time_to_first_token.observe(labels, ttft)
121 + for tpot in stats.time_per_output_tokens:
122 + histogram_time_per_output_tokens.observe(labels, tpot)
123 + for e2e in stats.time_e2e_requests:
124 + histogram_e2e_request_latency.observe(labels, e2e)
125 +
126 + def _log_prometheus_interval(self, prompt_throughput: float,
127 + generation_throughput: float) -> None:
128 + # Logs metrics to prometheus that are computed every logging_interval.
129 + # Support legacy gauge metrics that make throughput calculations on the vLLM side.
130 + # Moving forward, we should use counters like counter_prompt_tokens, counter_generation_tokens
131 + # Which log raw data and calculate summaries using rate() on the grafana/prometheus side.
132 + # See https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666
133 + gauge_avg_prompt_throughput.set(labels, prompt_throughput)
134 + gauge_avg_generation_throughput.set(labels, generation_throughput)
135 +
136 + def log(self, stats: Stats) -> None:
137 + """Called by LLMEngine.
138 + Logs to prometheus and tracked stats every iteration.
139 + Logs to Stdout every self.local_interval seconds."""
140 +
141 + # Log to prometheus.
142 + self._log_prometheus(stats)
143 +
144 + # Save tracked stats for token counters.
145 + self.num_prompt_tokens.append(stats.num_prompt_tokens)
146 + self.num_generation_tokens.append(stats.num_generation_tokens)
147 +
148 + # Log locally every local_interval seconds.
149 + if self._local_interval_elapsed(stats.now):
150 +
151 + # Compute summary metrics for tracked stats (and log them to promethus if applicable).
152 + prompt_throughput = self._get_throughput(self.num_prompt_tokens,
153 + now=stats.now)
154 + generation_throughput = self._get_throughput(
155 + self.num_generation_tokens, now=stats.now)
156 + self._log_prometheus_interval(
157 + prompt_throughput=prompt_throughput,
158 + generation_throughput=generation_throughput)
159
160 + # Log to stdout.
161 + logger.info(
162 + f"Avg prompt throughput: {prompt_throughput:.1f} tokens/s, "
163 + f"Avg generation throughput: {generation_throughput:.1f} tokens/s, "
164 + f"Running: {stats.num_running} reqs, "
165 + f"Swapped: {stats.num_swapped} reqs, "
166 + f"Pending: {stats.num_waiting} reqs, "
167 + f"GPU KV cache usage: {stats.gpu_cache_usage * 100:.1f}%, "
168 + f"CPU KV cache usage: {stats.cpu_cache_usage * 100:.1f}%")
169
36 -def record_metrics(
37 - avg_prompt_throughput: float,
38 - avg_generation_throughput: float,
39 - scheduler_running: int,
40 - scheduler_swapped: int,
41 - scheduler_waiting: int,
42 - gpu_cache_usage: float,
43 - cpu_cache_usage: float,
44 -):
45 - gauge_avg_prompt_throughput.set(labels, avg_prompt_throughput)
46 - gauge_avg_generation_throughput.set(labels, avg_generation_throughput)
47 - gauge_scheduler_running.set(labels, scheduler_running)
48 - gauge_scheduler_swapped.set(labels, scheduler_swapped)
49 - gauge_scheduler_waiting.set(labels, scheduler_waiting)
50 - gauge_gpu_cache_usage.set(labels, gpu_cache_usage)
51 - gauge_cpu_cache_usage.set(labels, cpu_cache_usage)
170 + # Reset tracked stats for next interval.
171 + self.num_prompt_tokens = []
172 + self.num_generation_tokens = []
173 + self.last_local_log = stats.now
vllm/sequence.py
+7 -1
@@ -52,7 +52,6 @@ class SequenceStatus(enum.Enum):
52 class SequenceData:
53 """Data associated with a sequence.
54
55 -
55 Args:
56 prompt_token_ids: The token IDs of the prompt.
57
@@ -254,6 +253,7 @@ class SequenceGroup:
253 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
254 self.sampling_params = sampling_params
255 self.arrival_time = arrival_time
256 + self.last_token_time = arrival_time
257 self.lora_request = lora_request
258 self.prefix: Optional[Prefix] = prefix
259 self.prompt_logprobs: Optional[PromptLogprobs] = None
@@ -274,6 +274,12 @@ class SequenceGroup:
274 def lora_int_id(self) -> int:
275 return self.lora_request.lora_int_id if self.lora_request else 0
276
277 + def get_last_latency(self, now: float) -> float:
278 + """Gets last token latency for Request level timings."""
279 + latency = now - self.last_token_time
280 + self.last_token_time = now
281 + return latency
282 +
283 def get_max_num_running_seqs(self) -> int:
284 """The maximum number of sequences running in parallel in the remaining
285 lifetime of the request."""