A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
Add more Prometheus metrics (#2764)
Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com> Co-authored-by: Robert Shaw <rshaw@neuralmagic.com>
Ronen Schaffer committed
Apr 29, 2024 at 01:59 UTC
bf480c53027cb009427581af0100de75ac7a2e5f
6 files changed
+582
-114
examples/production_monitoring/grafana.json
+283
@@ -873,6 +873,289 @@
873
],
874
"title": "Cache Utilization",
875
"type": "timeseries"
876
+ },
877
+ {
878
+ "type": "heatmap",
879
+ "title": "Request Prompt Length",
880
+ "description": "Heatmap of request prompt length",
881
+ "gridPos": {
882
+ "x": 0,
883
+ "y": 24,
884
+ "w": 12,
885
+ "h": 8
886
+ },
887
+ "datasource": {
888
+ "uid": "prometheus",
889
+ "type": "prometheus"
890
+ },
891
+ "id": 12,
892
+ "targets": [
893
+ {
894
+ "datasource": {
895
+ "type": "prometheus",
896
+ "uid": "prometheus"
897
+ },
898
+ "refId": "A",
899
+ "expr": "sum by(le) (increase(vllm:request_prompt_tokens_bucket{model_name=\"$model_name\"}[$__rate_interval]))",
900
+ "range": true,
901
+ "instant": false,
902
+ "editorMode": "builder",
903
+ "legendFormat": "{{le}}",
904
+ "useBackend": false,
905
+ "disableTextWrap": false,
906
+ "fullMetaSearch": false,
907
+ "includeNullMetadata": true,
908
+ "format": "heatmap"
909
+ }
910
+ ],
911
+ "options": {
912
+ "calculate": false,
913
+ "yAxis": {
914
+ "axisPlacement": "left",
915
+ "reverse": false,
916
+ "unit": "none",
917
+ "axisLabel": "Prompt Length"
918
+ },
919
+ "rowsFrame": {
920
+ "layout": "auto",
921
+ "value": "Request count"
922
+ },
923
+ "color": {
924
+ "mode": "scheme",
925
+ "fill": "dark-orange",
926
+ "scale": "exponential",
927
+ "exponent": 0.5,
928
+ "scheme": "Spectral",
929
+ "steps": 64,
930
+ "reverse": false,
931
+ "min": 0
932
+ },
933
+ "cellGap": 1,
934
+ "filterValues": {
935
+ "le": 1e-9
936
+ },
937
+ "tooltip": {
938
+ "show": true,
939
+ "yHistogram": true
940
+ },
941
+ "legend": {
942
+ "show": true
943
+ },
944
+ "exemplars": {
945
+ "color": "rgba(255,0,255,0.7)"
946
+ },
947
+ "cellValues": {
948
+ "unit": "none"
949
+ }
950
+ },
951
+ "fieldConfig": {
952
+ "defaults": {
953
+ "custom": {
954
+ "scaleDistribution": {
955
+ "type": "linear"
956
+ },
957
+ "hideFrom": {
958
+ "tooltip": false,
959
+ "viz": false,
960
+ "legend": false
961
+ }
962
+ }
963
+ },
964
+ "overrides": []
965
+ },
966
+ "pluginVersion": "10.2.0"
967
+ },
968
+ {
969
+ "datasource": {
970
+ "uid": "prometheus",
971
+ "type": "prometheus"
972
+ },
973
+ "type": "heatmap",
974
+ "title": "Request Generation Length",
975
+ "description": "Heatmap of request generation length",
976
+ "gridPos": {
977
+ "x": 12,
978
+ "y": 24,
979
+ "w": 12,
980
+ "h": 8
981
+ },
982
+ "id": 13,
983
+ "targets": [
984
+ {
985
+ "datasource": {
986
+ "type": "prometheus",
987
+ "uid": "prometheus"
988
+ },
989
+ "refId": "A",
990
+ "expr": "sum by(le) (increase(vllm:request_generation_tokens_bucket{model_name=\"$model_name\"}[$__rate_interval]))",
991
+ "range": true,
992
+ "instant": false,
993
+ "editorMode": "builder",
994
+ "legendFormat": "{{le}}",
995
+ "useBackend": false,
996
+ "disableTextWrap": false,
997
+ "fullMetaSearch": false,
998
+ "includeNullMetadata": true,
999
+ "format": "heatmap"
1000
+ }
1001
+ ],
1002
+ "options": {
1003
+ "calculate": false,
1004
+ "yAxis": {
1005
+ "axisPlacement": "left",
1006
+ "reverse": false,
1007
+ "unit": "none",
1008
+ "axisLabel": "Generation Length"
1009
+ },
1010
+ "rowsFrame": {
1011
+ "layout": "auto",
1012
+ "value": "Request count"
1013
+ },
1014
+ "color": {
1015
+ "mode": "scheme",
1016
+ "fill": "dark-orange",
1017
+ "scale": "exponential",
1018
+ "exponent": 0.5,
1019
+ "scheme": "Spectral",
1020
+ "steps": 64,
1021
+ "reverse": false,
1022
+ "min": 0
1023
+ },
1024
+ "cellGap": 1,
1025
+ "filterValues": {
1026
+ "le": 1e-9
1027
+ },
1028
+ "tooltip": {
1029
+ "show": true,
1030
+ "yHistogram": true
1031
+ },
1032
+ "legend": {
1033
+ "show": true
1034
+ },
1035
+ "exemplars": {
1036
+ "color": "rgba(255,0,255,0.7)"
1037
+ },
1038
+ "cellValues": {
1039
+ "unit": "none"
1040
+ }
1041
+ },
1042
+ "fieldConfig": {
1043
+ "defaults": {
1044
+ "custom": {
1045
+ "scaleDistribution": {
1046
+ "type": "linear"
1047
+ },
1048
+ "hideFrom": {
1049
+ "tooltip": false,
1050
+ "viz": false,
1051
+ "legend": false
1052
+ }
1053
+ }
1054
+ },
1055
+ "overrides": []
1056
+ },
1057
+ "pluginVersion": "10.2.0"
1058
+ },
1059
+ {
1060
+ "datasource": {
1061
+ "type": "prometheus",
1062
+ "uid": "prometheus"
1063
+ },
1064
+ "fieldConfig": {
1065
+ "defaults": {
1066
+ "custom": {
1067
+ "drawStyle": "line",
1068
+ "lineInterpolation": "linear",
1069
+ "barAlignment": 0,
1070
+ "lineWidth": 1,
1071
+ "fillOpacity": 0,
1072
+ "gradientMode": "none",
1073
+ "spanNulls": false,
1074
+ "insertNulls": false,
1075
+ "showPoints": "auto",
1076
+ "pointSize": 5,
1077
+ "stacking": {
1078
+ "mode": "none",
1079
+ "group": "A"
1080
+ },
1081
+ "axisPlacement": "auto",
1082
+ "axisLabel": "",
1083
+ "axisColorMode": "text",
1084
+ "axisBorderShow": false,
1085
+ "scaleDistribution": {
1086
+ "type": "linear"
1087
+ },
1088
+ "axisCenteredZero": false,
1089
+ "hideFrom": {
1090
+ "tooltip": false,
1091
+ "viz": false,
1092
+ "legend": false
1093
+ },
1094
+ "thresholdsStyle": {
1095
+ "mode": "off"
1096
+ }
1097
+ },
1098
+ "color": {
1099
+ "mode": "palette-classic"
1100
+ },
1101
+ "mappings": [],
1102
+ "thresholds": {
1103
+ "mode": "absolute",
1104
+ "steps": [
1105
+ {
1106
+ "color": "green",
1107
+ "value": null
1108
+ },
1109
+ {
1110
+ "color": "red",
1111
+ "value": 80
1112
+ }
1113
+ ]
1114
+ }
1115
+ },
1116
+ "overrides": []
1117
+ },
1118
+ "gridPos": {
1119
+ "h": 8,
1120
+ "w": 12,
1121
+ "x": 0,
1122
+ "y": 32
1123
+ },
1124
+ "id": 11,
1125
+ "options": {
1126
+ "tooltip": {
1127
+ "mode": "single",
1128
+ "sort": "none"
1129
+ },
1130
+ "legend": {
1131
+ "showLegend": true,
1132
+ "displayMode": "list",
1133
+ "placement": "bottom",
1134
+ "calcs": []
1135
+ }
1136
+ },
1137
+ "targets": [
1138
+ {
1139
+ "datasource": {
1140
+ "type": "prometheus",
1141
+ "uid": "prometheus"
1142
+ },
1143
+ "disableTextWrap": false,
1144
+ "editorMode": "builder",
1145
+ "expr": "sum by(finished_reason) (increase(vllm:request_success_total{model_name=\"$model_name\"}[$__rate_interval]))",
1146
+ "fullMetaSearch": false,
1147
+ "includeNullMetadata": true,
1148
+ "instant": false,
1149
+ "interval": "",
1150
+ "legendFormat": "__auto",
1151
+ "range": true,
1152
+ "refId": "A",
1153
+ "useBackend": false
1154
+ }
1155
+ ],
1156
+ "title": "Finish Reason",
1157
+ "description": "Number of finished requests by their finish reason: either an EOS token was generated or the max sequence length was reached.",
1158
+ "type": "timeseries"
1159
}
1160
],
1161
"refresh": "",
requirements-common.txt
+1
@@ -12,6 +12,7 @@ openai
12
uvicorn[standard]
13
pydantic >= 2.0 # Required for OpenAI server.
14
prometheus_client >= 0.18.0
15
+prometheus-fastapi-instrumentator >= 7.0.0
16
tiktoken == 0.6.0 # Required for DBRX tokenizer
17
lm-format-enforcer == 0.9.8
18
outlines == 0.0.34 # Requires torch >= 2.1.0
vllm/core/scheduler.py
+1
-1
@@ -320,7 +320,7 @@ class Scheduler:
320
for seq_group in state_queue:
321
if not request_ids:
322
# Using 'break' here may add two extra iterations,
323
- # but is acceptable to reduce complexity .
323
+ # but is acceptable to reduce complexity.
324
break
325
if seq_group.request_id in request_ids:
326
# Appending aborted group into pending list.
vllm/engine/llm_engine.py
+119
-52
@@ -22,7 +22,8 @@ from vllm.lora.request import LoRARequest
22
from vllm.outputs import RequestOutput
23
from vllm.sampling_params import SamplingParams
24
from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
25
- SequenceGroup, SequenceGroupMetadata)
25
+ SequenceGroup, SequenceGroupMetadata,
26
+ SequenceStatus)
27
from vllm.transformers_utils.detokenizer import Detokenizer
28
from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
29
get_tokenizer_group)
@@ -217,7 +218,8 @@ class LLMEngine:
218
if self.log_stats:
219
self.stat_logger = StatLogger(
220
local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
220
- labels=dict(model_name=model_config.model))
221
+ labels=dict(model_name=model_config.model),
222
+ max_model_len=self.model_config.max_model_len)
223
self.stat_logger.info("cache_config", self.cache_config)
224
225
# Create sequence output processor, e.g. for beam search or
@@ -619,59 +621,109 @@ class LLMEngine:
621
"""
622
now = time.time()
623
622
- # KV Cache Usage in %.
624
+ # System State
625
+ # Scheduler State
626
+ num_running_sys = len(self.scheduler.running)
627
+ num_swapped_sys = len(self.scheduler.swapped)
628
+ num_waiting_sys = len(self.scheduler.waiting)
629
+
630
+ # KV Cache Usage in %
631
num_total_gpu = self.cache_config.num_gpu_blocks
632
num_free_gpu = self.scheduler.block_manager.get_num_free_gpu_blocks()
625
- gpu_cache_usage = 1.0 - (num_free_gpu / num_total_gpu)
633
+ gpu_cache_usage_sys = 1.0 - (num_free_gpu / num_total_gpu)
634
635
num_total_cpu = self.cache_config.num_cpu_blocks
628
- cpu_cache_usage = 0.
636
+ cpu_cache_usage_sys = 0.
637
if num_total_cpu > 0:
638
num_free_cpu = self.scheduler.block_manager.get_num_free_cpu_blocks(
639
)
632
- cpu_cache_usage = 1.0 - (num_free_cpu / num_total_cpu)
633
-
634
- # Scheduler State
635
- num_running = len(self.scheduler.running)
636
- num_swapped = len(self.scheduler.swapped)
637
- num_waiting = len(self.scheduler.waiting)
638
-
639
- # Iteration stats if we have scheduler output.
640
- num_prompt_tokens = 0
641
- num_generation_tokens = 0
642
- time_to_first_tokens = []
643
- time_per_output_tokens = []
644
- time_e2e_requests = []
640
+ cpu_cache_usage_sys = 1.0 - (num_free_cpu / num_total_cpu)
641
+
642
+ # Iteration stats
643
+ num_prompt_tokens_iter = 0
644
+ num_generation_tokens_iter = 0
645
+ time_to_first_tokens_iter: List[float] = []
646
+ time_per_output_tokens_iter: List[float] = []
647
+
648
+ # Request stats
649
+ # Latency
650
+ time_e2e_requests: List[float] = []
651
+ # Metadata
652
+ num_prompt_tokens_requests: List[int] = []
653
+ num_generation_tokens_requests: List[int] = []
654
+ best_of_requests: List[int] = []
655
+ n_requests: List[int] = []
656
+ finished_reason_requests: List[str] = []
657
+
658
+ # NOTE: This loop assumes prefill seq_groups are before
659
+ # decode seq_groups in scheduled_seq_groups.
660
if scheduler_outputs is not None:
646
- prompt_run = scheduler_outputs.num_prefill_groups > 0
647
-
648
- # Number of Tokens.
649
- if prompt_run:
650
- num_prompt_tokens = sum(
651
- len(scheduled_seq_group.seq_group.prompt_token_ids)
652
- for scheduled_seq_group in
653
- scheduler_outputs.scheduled_seq_groups)
654
- num_generation_tokens = sum(
655
- scheduled_seq_group.seq_group.num_seqs()
656
- for scheduled_seq_group in
657
- scheduler_outputs.scheduled_seq_groups)
658
- else:
659
- num_generation_tokens = scheduler_outputs.num_batched_tokens
660
-
661
- # Latency Timings.
662
- time_last_iters = []
663
- for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
661
+ num_generation_tokens_from_prefill_groups = 0.
662
+ if scheduler_outputs.num_prefill_groups > 0 and len(
663
+ scheduler_outputs.scheduled_seq_groups
664
+ ) != scheduler_outputs.num_prefill_groups:
665
+ print("DETECTED CHUNKED")
666
+
667
+ for idx, scheduled_seq_group in enumerate(
668
+ scheduler_outputs.scheduled_seq_groups):
669
+ group_was_prefill = idx < scheduler_outputs.num_prefill_groups
670
seq_group = scheduled_seq_group.seq_group
665
- # Time since last token.
666
- # (n.b. updates seq_group.metrics.last_token_time)
667
- time_last_iters.append(seq_group.get_last_latency(now))
668
- # Time since arrival for all finished requests.
671
+
672
+ # NOTE: a seq_group that completed all of its prefill tokens
673
+ # in the last iteration will have seq_group.is_prefill() = False
674
+ # with group_was_prefill = True
675
+ if group_was_prefill:
676
+ # Number of prompt tokens.
677
+ num_prompt_tokens_iter += (
678
+ scheduled_seq_group.token_chunk_size)
679
+
680
+ # If the seq_group just finished the prefill state
681
+ # get TTFT.
682
+ if not seq_group.is_prefill():
683
+ latency = seq_group.get_last_latency(now)
684
+ time_to_first_tokens_iter.append(latency)
685
+
686
+ # One generation token per finished prefill.
687
+ num_generation_tokens_from_prefill_groups += (
688
+ seq_group.num_seqs())
689
+ else:
690
+ # TPOTs.
691
+ latency = seq_group.get_last_latency(now)
692
+ time_per_output_tokens_iter.append(latency)
693
+
694
+ # Because of chunked prefill, we can have a single sequence
695
+ # group that does multiple prompt_runs. To prevent logging
696
+ # the same metadata more than once per request, we standardize
697
+ # on logging request level information for finished requests,
698
+ # which can only happen once.
699
if seq_group.is_finished():
700
+ # Latency timings
701
time_e2e_requests.append(now -
702
seq_group.metrics.arrival_time)
703
673
- time_to_first_tokens = time_last_iters if prompt_run else []
674
- time_per_output_tokens = [] if prompt_run else time_last_iters
704
+ # Metadata
705
+ num_prompt_tokens_requests.append(
706
+ len(seq_group.prompt_token_ids))
707
+ num_generation_tokens_requests.extend([
708
+ seq.get_output_len()
709
+ for seq in seq_group.get_finished_seqs()
710
+ ])
711
+ best_of_requests.append(seq_group.sampling_params.best_of)
712
+ n_requests.append(seq_group.sampling_params.n)
713
+ finished_reason_requests.extend([
714
+ SequenceStatus.get_finished_reason(seq.status)
715
+ for seq in seq_group.get_finished_seqs()
716
+ ])
717
+
718
+ # Number of generation tokens.
719
+ # num_batched_tokens equals the number of prompt_tokens plus the
720
+ # number of decode_tokens in a single iteration. So,
721
+ # num_generation_tokens = num_batched_tokens - num_prompt_tokens
722
+ # + num_generation_tokens_from_prefill_groups (since we generate
723
+ # one token on prefills on iters where the prefill finishes).
724
+ num_generation_tokens_iter = (
725
+ scheduler_outputs.num_batched_tokens - num_prompt_tokens_iter +
726
+ num_generation_tokens_from_prefill_groups)
727
728
# Spec decode, if enabled, emits specialized metrics from the worker in
729
# sampler output.
@@ -683,17 +735,32 @@ class LLMEngine:
735
736
return Stats(
737
now=now,
686
- num_running=num_running,
687
- num_swapped=num_swapped,
688
- num_waiting=num_waiting,
689
- gpu_cache_usage=gpu_cache_usage,
690
- cpu_cache_usage=cpu_cache_usage,
691
- num_prompt_tokens=num_prompt_tokens,
692
- num_generation_tokens=num_generation_tokens,
693
- time_to_first_tokens=time_to_first_tokens,
694
- time_per_output_tokens=time_per_output_tokens,
695
- time_e2e_requests=time_e2e_requests,
738
+
739
+ # System stats
740
+ # Scheduler State
741
+ num_running_sys=num_running_sys,
742
+ num_swapped_sys=num_swapped_sys,
743
+ num_waiting_sys=num_waiting_sys,
744
+ # KV Cache Usage in %
745
+ gpu_cache_usage_sys=gpu_cache_usage_sys,
746
+ cpu_cache_usage_sys=cpu_cache_usage_sys,
747
+
748
+ # Iteration stats
749
+ num_prompt_tokens_iter=num_prompt_tokens_iter,
750
+ num_generation_tokens_iter=num_generation_tokens_iter,
751
+ time_to_first_tokens_iter=time_to_first_tokens_iter,
752
+ time_per_output_tokens_iter=time_per_output_tokens_iter,
753
spec_decode_metrics=spec_decode_metrics,
754
+
755
+ # Request stats
756
+ # Latency
757
+ time_e2e_requests=time_e2e_requests,
758
+ # Metadata
759
+ num_prompt_tokens_requests=num_prompt_tokens_requests,
760
+ num_generation_tokens_requests=num_generation_tokens_requests,
761
+ best_of_requests=best_of_requests,
762
+ n_requests=n_requests,
763
+ finished_reason_requests=finished_reason_requests,
764
)
765
766
def add_lora(self, lora_request: LoRARequest) -> bool:
vllm/engine/metrics.py
+163
-58
@@ -1,6 +1,8 @@
1
import time
2
from dataclasses import dataclass
3
-from typing import TYPE_CHECKING, Dict, List, Optional, Protocol
3
+from typing import TYPE_CHECKING
4
+from typing import Counter as CollectionsCounter
5
+from typing import Dict, List, Optional, Protocol, Union
6
7
import numpy as np
8
from prometheus_client import (REGISTRY, Counter, Gauge, Histogram, Info,
@@ -21,8 +23,9 @@ disable_created_metrics()
23
24
# begin-metrics-definitions
25
class Metrics:
26
+ labelname_finish_reason = "finished_reason"
27
25
- def __init__(self, labelnames: List[str]):
28
+ def __init__(self, labelnames: List[str], max_model_len: int):
29
# Unregister any existing vLLM collectors
30
for collector in list(REGISTRY._collector_to_names):
31
if hasattr(collector, "_name") and "vllm" in collector._name:
@@ -34,18 +37,20 @@ class Metrics:
37
documentation='information of cache_config')
38
39
# System stats
40
+ # Scheduler State
41
self.gauge_scheduler_running = Gauge(
42
name="vllm:num_requests_running",
43
documentation="Number of requests currently running on GPU.",
44
labelnames=labelnames)
41
- self.gauge_scheduler_swapped = Gauge(
42
- name="vllm:num_requests_swapped",
43
- documentation="Number of requests swapped to CPU.",
44
- labelnames=labelnames)
45
self.gauge_scheduler_waiting = Gauge(
46
name="vllm:num_requests_waiting",
47
documentation="Number of requests waiting to be processed.",
48
labelnames=labelnames)
49
+ self.gauge_scheduler_swapped = Gauge(
50
+ name="vllm:num_requests_swapped",
51
+ documentation="Number of requests swapped to CPU.",
52
+ labelnames=labelnames)
53
+ # KV Cache Usage in %
54
self.gauge_gpu_cache_usage = Gauge(
55
name="vllm:gpu_cache_usage_perc",
56
documentation="GPU KV-cache usage. 1 means 100 percent usage.",
@@ -55,7 +60,7 @@ class Metrics:
60
documentation="CPU KV-cache usage. 1 means 100 percent usage.",
61
labelnames=labelnames)
62
58
- # Raw stats from last model iteration
63
+ # Iteration stats
64
self.counter_prompt_tokens = Counter(
65
name="vllm:prompt_tokens_total",
66
documentation="Number of prefill tokens processed.",
@@ -80,18 +85,51 @@ class Metrics:
85
0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
86
1.0, 2.5
87
])
83
- self.histogram_e2e_request_latency = Histogram(
88
+
89
+ # Request stats
90
+ # Latency
91
+ self.histogram_e2e_time_request = Histogram(
92
name="vllm:e2e_request_latency_seconds",
93
documentation="Histogram of end to end request latency in seconds.",
94
labelnames=labelnames,
95
buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
96
+ # Metadata
97
+ self.histogram_num_prompt_tokens_request = Histogram(
98
+ name="vllm:request_prompt_tokens",
99
+ documentation="Number of prefill tokens processed.",
100
+ labelnames=labelnames,
101
+ buckets=build_1_2_5_buckets(max_model_len),
102
+ )
103
+ self.histogram_num_generation_tokens_request = Histogram(
104
+ name="vllm:request_generation_tokens",
105
+ documentation="Number of generation tokens processed.",
106
+ labelnames=labelnames,
107
+ buckets=build_1_2_5_buckets(max_model_len),
108
+ )
109
+ self.histogram_best_of_request = Histogram(
110
+ name="vllm:request_params_best_of",
111
+ documentation="Histogram of the best_of request parameter.",
112
+ labelnames=labelnames,
113
+ buckets=[1, 2, 5, 10, 20],
114
+ )
115
+ self.histogram_n_request = Histogram(
116
+ name="vllm:request_params_n",
117
+ documentation="Histogram of the n request parameter.",
118
+ labelnames=labelnames,
119
+ buckets=[1, 2, 5, 10, 20],
120
+ )
121
+ self.counter_request_success = Counter(
122
+ name="vllm:request_success",
123
+ documentation="Count of successfully processed requests.",
124
+ labelnames=labelnames + [Metrics.labelname_finish_reason])
125
89
- # Legacy metrics
126
+ # Deprecated in favor of vllm:prompt_tokens_total
127
self.gauge_avg_prompt_throughput = Gauge(
128
name="vllm:avg_prompt_throughput_toks_per_s",
129
documentation="Average prefill throughput in tokens/s.",
130
labelnames=labelnames,
131
)
132
+ # Deprecated in favor of vllm:generation_tokens_total
133
self.gauge_avg_generation_throughput = Gauge(
134
name="vllm:avg_generation_throughput_toks_per_s",
135
documentation="Average generation throughput in tokens/s.",
@@ -102,24 +140,57 @@ class Metrics:
140
# end-metrics-definitions
141
142
143
+def build_1_2_5_buckets(max_value: int):
144
+ """
145
+ Builds a list of buckets with increasing powers of 10 multiplied by
146
+ mantissa values (1, 2, 5) until the value exceeds the specified maximum.
147
+
148
+ Example:
149
+ >>> build_1_2_5_buckets(100)
150
+ [1, 2, 5, 10, 20, 50, 100]
151
+ """
152
+ mantissa_lst = [1, 2, 5]
153
+ exponent = 0
154
+ buckets = []
155
+ while True:
156
+ for m in mantissa_lst:
157
+ value = m * 10**exponent
158
+ if value <= max_value:
159
+ buckets.append(value)
160
+ else:
161
+ return buckets
162
+ exponent += 1
163
+
164
+
165
@dataclass
166
class Stats:
167
"""Created by LLMEngine for use by StatLogger."""
168
now: float
169
110
- # System stats.
111
- num_running: int
112
- num_waiting: int
113
- num_swapped: int
114
- gpu_cache_usage: float
115
- cpu_cache_usage: float
116
-
117
- # Raw stats from last model iteration.
118
- num_prompt_tokens: int
119
- num_generation_tokens: int
120
- time_to_first_tokens: List[float]
121
- time_per_output_tokens: List[float]
170
+ # System stats (should have _sys suffix)
171
+ # Scheduler State
172
+ num_running_sys: int
173
+ num_waiting_sys: int
174
+ num_swapped_sys: int
175
+ # KV Cache Usage in %
176
+ gpu_cache_usage_sys: float
177
+ cpu_cache_usage_sys: float
178
+
179
+ # Iteration stats (should have _iter suffix)
180
+ num_prompt_tokens_iter: int
181
+ num_generation_tokens_iter: int
182
+ time_to_first_tokens_iter: List[float]
183
+ time_per_output_tokens_iter: List[float]
184
+
185
+ # Request stats (should have _requests suffix)
186
+ # Latency
187
time_e2e_requests: List[float]
188
+ # Metadata
189
+ num_prompt_tokens_requests: List[int]
190
+ num_generation_tokens_requests: List[int]
191
+ best_of_requests: List[int]
192
+ n_requests: List[int]
193
+ finished_reason_requests: List[str]
194
195
spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
196
@@ -133,7 +204,8 @@ class SupportsMetricsInfo(Protocol):
204
class StatLogger:
205
"""StatLogger is used LLMEngine to log to Promethus and Stdout."""
206
136
- def __init__(self, local_interval: float, labels: Dict[str, str]) -> None:
207
+ def __init__(self, local_interval: float, labels: Dict[str, str],
208
+ max_model_len: int) -> None:
209
# Metadata for logging locally.
210
self.last_local_log = time.time()
211
self.local_interval = local_interval
@@ -144,7 +216,8 @@ class StatLogger:
216
217
# Prometheus metrics
218
self.labels = labels
147
- self.metrics = Metrics(labelnames=list(labels.keys()))
219
+ self.metrics = Metrics(labelnames=list(labels.keys()),
220
+ max_model_len=max_model_len)
221
222
def info(self, type: str, obj: SupportsMetricsInfo) -> None:
223
if type == "cache_config":
@@ -158,34 +231,66 @@ class StatLogger:
231
return elapsed_time > self.local_interval
232
233
def _log_prometheus(self, stats: Stats) -> None:
161
- # Set system stat gauges.
162
- self.metrics.gauge_scheduler_running.labels(**self.labels).set(
163
- stats.num_running)
164
- self.metrics.gauge_scheduler_swapped.labels(**self.labels).set(
165
- stats.num_swapped)
166
- self.metrics.gauge_scheduler_waiting.labels(**self.labels).set(
167
- stats.num_waiting)
168
- self.metrics.gauge_gpu_cache_usage.labels(**self.labels).set(
169
- stats.gpu_cache_usage)
170
- self.metrics.gauge_cpu_cache_usage.labels(**self.labels).set(
171
- stats.cpu_cache_usage)
172
-
173
- # Add to token counters.
174
- self.metrics.counter_prompt_tokens.labels(**self.labels).inc(
175
- stats.num_prompt_tokens)
176
- self.metrics.counter_generation_tokens.labels(**self.labels).inc(
177
- stats.num_generation_tokens)
178
-
179
- # Observe request level latencies in histograms.
180
- for ttft in stats.time_to_first_tokens:
181
- self.metrics.histogram_time_to_first_token.labels(
182
- **self.labels).observe(ttft)
183
- for tpot in stats.time_per_output_tokens:
184
- self.metrics.histogram_time_per_output_token.labels(
185
- **self.labels).observe(tpot)
186
- for e2e in stats.time_e2e_requests:
187
- self.metrics.histogram_e2e_request_latency.labels(
188
- **self.labels).observe(e2e)
234
+ # System state data
235
+ self._log_gauge(self.metrics.gauge_scheduler_running,
236
+ stats.num_running_sys)
237
+ self._log_gauge(self.metrics.gauge_scheduler_swapped,
238
+ stats.num_swapped_sys)
239
+ self._log_gauge(self.metrics.gauge_scheduler_waiting,
240
+ stats.num_waiting_sys)
241
+ self._log_gauge(self.metrics.gauge_gpu_cache_usage,
242
+ stats.gpu_cache_usage_sys)
243
+ self._log_gauge(self.metrics.gauge_cpu_cache_usage,
244
+ stats.cpu_cache_usage_sys)
245
+
246
+ # Iteration level data
247
+ self._log_counter(self.metrics.counter_prompt_tokens,
248
+ stats.num_prompt_tokens_iter)
249
+ self._log_counter(self.metrics.counter_generation_tokens,
250
+ stats.num_generation_tokens_iter)
251
+ self._log_histogram(self.metrics.histogram_time_to_first_token,
252
+ stats.time_to_first_tokens_iter)
253
+ self._log_histogram(self.metrics.histogram_time_per_output_token,
254
+ stats.time_per_output_tokens_iter)
255
+
256
+ # Request level data
257
+ # Latency
258
+ self._log_histogram(self.metrics.histogram_e2e_time_request,
259
+ stats.time_e2e_requests)
260
+ # Metadata
261
+ finished_reason_counter = CollectionsCounter(
262
+ stats.finished_reason_requests)
263
+ self._log_counter_labels(self.metrics.counter_request_success,
264
+ finished_reason_counter,
265
+ Metrics.labelname_finish_reason)
266
+ self._log_histogram(self.metrics.histogram_num_prompt_tokens_request,
267
+ stats.num_prompt_tokens_requests)
268
+ self._log_histogram(
269
+ self.metrics.histogram_num_generation_tokens_request,
270
+ stats.num_generation_tokens_requests)
271
+ self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
272
+ self._log_histogram(self.metrics.histogram_best_of_request,
273
+ stats.best_of_requests)
274
+
275
+ def _log_gauge(self, gauge: Gauge, data: Union[int, float]) -> None:
276
+ # Convenience function for logging to gauge.
277
+ gauge.labels(**self.labels).set(data)
278
+
279
+ def _log_counter(self, counter: Counter, data: Union[int, float]) -> None:
280
+ # Convenience function for logging to counter.
281
+ counter.labels(**self.labels).inc(data)
282
+
283
+ def _log_counter_labels(self, counter: Counter, data: CollectionsCounter,
284
+ label_key: str) -> None:
285
+ # Convenience function for collection counter of labels.
286
+ for label, count in data.items():
287
+ counter.labels(**{**self.labels, label_key: label}).inc(count)
288
+
289
+ def _log_histogram(self, histogram: Histogram,
290
+ data: Union[List[int], List[float]]) -> None:
291
+ # Convenience function for logging list to histogram.
292
+ for datum in data:
293
+ histogram.labels(**self.labels).observe(datum)
294
295
def _log_prometheus_interval(self, prompt_throughput: float,
296
generation_throughput: float) -> None:
@@ -210,8 +315,8 @@ class StatLogger:
315
self._log_prometheus(stats)
316
317
# Save tracked stats for token counters.
213
- self.num_prompt_tokens.append(stats.num_prompt_tokens)
214
- self.num_generation_tokens.append(stats.num_generation_tokens)
318
+ self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
319
+ self.num_generation_tokens.append(stats.num_generation_tokens_iter)
320
321
# Log locally every local_interval seconds.
322
if self._local_interval_elapsed(stats.now):
@@ -234,11 +339,11 @@ class StatLogger:
339
"CPU KV cache usage: %.1f%%",
340
prompt_throughput,
341
generation_throughput,
237
- stats.num_running,
238
- stats.num_swapped,
239
- stats.num_waiting,
240
- stats.gpu_cache_usage * 100,
241
- stats.cpu_cache_usage * 100,
342
+ stats.num_running_sys,
343
+ stats.num_swapped_sys,
344
+ stats.num_waiting_sys,
345
+ stats.gpu_cache_usage_sys * 100,
346
+ stats.cpu_cache_usage_sys * 100,
347
)
348
349
# Reset tracked stats for next interval.
vllm/sequence.py
+15
-3
@@ -442,15 +442,27 @@ class SequenceGroup:
442
def lora_int_id(self) -> int:
443
return self.lora_request.lora_int_id if self.lora_request else 0
444
445
- def get_last_latency(self, now: float) -> float:
446
- """Gets last token latency for Request level timings."""
445
+ def get_last_latency(self, now: float) -> Optional[float]:
446
+ """Sets the last token time for Request level timings."""
447
+ # If still in prefill phase, raise Error.
448
+ if self.is_prefill():
449
+ raise ValueError(
450
+ "seq_group.get_last_latency() should not be called "
451
+ "if the seq_group is in prefill phase.")
452
+
453
+ # Otherwise return token latency.
454
latency = now - self.metrics.last_token_time
455
self.metrics.last_token_time = now
456
return latency
457
458
def maybe_set_first_token_time(self, time: float) -> None:
459
"""Sets the first token time for Request level timings."""
453
- if self.metrics.first_token_time is None:
460
+ # Note: in a case where a sequence_group is swapped and
461
+ # recomputed, the time between iterations is counted
462
+ # in TPOT, rather than recalculating TTFT (since from the )
463
+ # POV of the user, there is simply a long generation delay.
464
+ if (self.metrics.first_token_time is None
465
+ and self.get_seqs()[0].get_output_len() == 1):
466
self.metrics.first_token_time = time
467
468
def maybe_set_first_scheduled_time(self, time: float) -> None: