@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bf480c5302
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

Add more Prometheus metrics (#2764)

Co-authored-by: Robert Shaw <114415538+robertgshaw2-neuralmagic@users.noreply.github.com> Co-authored-by: Robert Shaw <rshaw@neuralmagic.com>

Ronen Schaffer committed Apr 29, 2024 at 01:59 UTC bf480c53027cb009427581af0100de75ac7a2e5f
6 files changed +582 -114
examples/production_monitoring/grafana.json
+283
@@ -873,6 +873,289 @@
873 ],
874 "title": "Cache Utilization",
875 "type": "timeseries"
876 + },
877 + {
878 + "type": "heatmap",
879 + "title": "Request Prompt Length",
880 + "description": "Heatmap of request prompt length",
881 + "gridPos": {
882 + "x": 0,
883 + "y": 24,
884 + "w": 12,
885 + "h": 8
886 + },
887 + "datasource": {
888 + "uid": "prometheus",
889 + "type": "prometheus"
890 + },
891 + "id": 12,
892 + "targets": [
893 + {
894 + "datasource": {
895 + "type": "prometheus",
896 + "uid": "prometheus"
897 + },
898 + "refId": "A",
899 + "expr": "sum by(le) (increase(vllm:request_prompt_tokens_bucket{model_name=\"$model_name\"}[$__rate_interval]))",
900 + "range": true,
901 + "instant": false,
902 + "editorMode": "builder",
903 + "legendFormat": "{{le}}",
904 + "useBackend": false,
905 + "disableTextWrap": false,
906 + "fullMetaSearch": false,
907 + "includeNullMetadata": true,
908 + "format": "heatmap"
909 + }
910 + ],
911 + "options": {
912 + "calculate": false,
913 + "yAxis": {
914 + "axisPlacement": "left",
915 + "reverse": false,
916 + "unit": "none",
917 + "axisLabel": "Prompt Length"
918 + },
919 + "rowsFrame": {
920 + "layout": "auto",
921 + "value": "Request count"
922 + },
923 + "color": {
924 + "mode": "scheme",
925 + "fill": "dark-orange",
926 + "scale": "exponential",
927 + "exponent": 0.5,
928 + "scheme": "Spectral",
929 + "steps": 64,
930 + "reverse": false,
931 + "min": 0
932 + },
933 + "cellGap": 1,
934 + "filterValues": {
935 + "le": 1e-9
936 + },
937 + "tooltip": {
938 + "show": true,
939 + "yHistogram": true
940 + },
941 + "legend": {
942 + "show": true
943 + },
944 + "exemplars": {
945 + "color": "rgba(255,0,255,0.7)"
946 + },
947 + "cellValues": {
948 + "unit": "none"
949 + }
950 + },
951 + "fieldConfig": {
952 + "defaults": {
953 + "custom": {
954 + "scaleDistribution": {
955 + "type": "linear"
956 + },
957 + "hideFrom": {
958 + "tooltip": false,
959 + "viz": false,
960 + "legend": false
961 + }
962 + }
963 + },
964 + "overrides": []
965 + },
966 + "pluginVersion": "10.2.0"
967 + },
968 + {
969 + "datasource": {
970 + "uid": "prometheus",
971 + "type": "prometheus"
972 + },
973 + "type": "heatmap",
974 + "title": "Request Generation Length",
975 + "description": "Heatmap of request generation length",
976 + "gridPos": {
977 + "x": 12,
978 + "y": 24,
979 + "w": 12,
980 + "h": 8
981 + },
982 + "id": 13,
983 + "targets": [
984 + {
985 + "datasource": {
986 + "type": "prometheus",
987 + "uid": "prometheus"
988 + },
989 + "refId": "A",
990 + "expr": "sum by(le) (increase(vllm:request_generation_tokens_bucket{model_name=\"$model_name\"}[$__rate_interval]))",
991 + "range": true,
992 + "instant": false,
993 + "editorMode": "builder",
994 + "legendFormat": "{{le}}",
995 + "useBackend": false,
996 + "disableTextWrap": false,
997 + "fullMetaSearch": false,
998 + "includeNullMetadata": true,
999 + "format": "heatmap"
1000 + }
1001 + ],
1002 + "options": {
1003 + "calculate": false,
1004 + "yAxis": {
1005 + "axisPlacement": "left",
1006 + "reverse": false,
1007 + "unit": "none",
1008 + "axisLabel": "Generation Length"
1009 + },
1010 + "rowsFrame": {
1011 + "layout": "auto",
1012 + "value": "Request count"
1013 + },
1014 + "color": {
1015 + "mode": "scheme",
1016 + "fill": "dark-orange",
1017 + "scale": "exponential",
1018 + "exponent": 0.5,
1019 + "scheme": "Spectral",
1020 + "steps": 64,
1021 + "reverse": false,
1022 + "min": 0
1023 + },
1024 + "cellGap": 1,
1025 + "filterValues": {
1026 + "le": 1e-9
1027 + },
1028 + "tooltip": {
1029 + "show": true,
1030 + "yHistogram": true
1031 + },
1032 + "legend": {
1033 + "show": true
1034 + },
1035 + "exemplars": {
1036 + "color": "rgba(255,0,255,0.7)"
1037 + },
1038 + "cellValues": {
1039 + "unit": "none"
1040 + }
1041 + },
1042 + "fieldConfig": {
1043 + "defaults": {
1044 + "custom": {
1045 + "scaleDistribution": {
1046 + "type": "linear"
1047 + },
1048 + "hideFrom": {
1049 + "tooltip": false,
1050 + "viz": false,
1051 + "legend": false
1052 + }
1053 + }
1054 + },
1055 + "overrides": []
1056 + },
1057 + "pluginVersion": "10.2.0"
1058 + },
1059 + {
1060 + "datasource": {
1061 + "type": "prometheus",
1062 + "uid": "prometheus"
1063 + },
1064 + "fieldConfig": {
1065 + "defaults": {
1066 + "custom": {
1067 + "drawStyle": "line",
1068 + "lineInterpolation": "linear",
1069 + "barAlignment": 0,
1070 + "lineWidth": 1,
1071 + "fillOpacity": 0,
1072 + "gradientMode": "none",
1073 + "spanNulls": false,
1074 + "insertNulls": false,
1075 + "showPoints": "auto",
1076 + "pointSize": 5,
1077 + "stacking": {
1078 + "mode": "none",
1079 + "group": "A"
1080 + },
1081 + "axisPlacement": "auto",
1082 + "axisLabel": "",
1083 + "axisColorMode": "text",
1084 + "axisBorderShow": false,
1085 + "scaleDistribution": {
1086 + "type": "linear"
1087 + },
1088 + "axisCenteredZero": false,
1089 + "hideFrom": {
1090 + "tooltip": false,
1091 + "viz": false,
1092 + "legend": false
1093 + },
1094 + "thresholdsStyle": {
1095 + "mode": "off"
1096 + }
1097 + },
1098 + "color": {
1099 + "mode": "palette-classic"
1100 + },
1101 + "mappings": [],
1102 + "thresholds": {
1103 + "mode": "absolute",
1104 + "steps": [
1105 + {
1106 + "color": "green",
1107 + "value": null
1108 + },
1109 + {
1110 + "color": "red",
1111 + "value": 80
1112 + }
1113 + ]
1114 + }
1115 + },
1116 + "overrides": []
1117 + },
1118 + "gridPos": {
1119 + "h": 8,
1120 + "w": 12,
1121 + "x": 0,
1122 + "y": 32
1123 + },
1124 + "id": 11,
1125 + "options": {
1126 + "tooltip": {
1127 + "mode": "single",
1128 + "sort": "none"
1129 + },
1130 + "legend": {
1131 + "showLegend": true,
1132 + "displayMode": "list",
1133 + "placement": "bottom",
1134 + "calcs": []
1135 + }
1136 + },
1137 + "targets": [
1138 + {
1139 + "datasource": {
1140 + "type": "prometheus",
1141 + "uid": "prometheus"
1142 + },
1143 + "disableTextWrap": false,
1144 + "editorMode": "builder",
1145 + "expr": "sum by(finished_reason) (increase(vllm:request_success_total{model_name=\"$model_name\"}[$__rate_interval]))",
1146 + "fullMetaSearch": false,
1147 + "includeNullMetadata": true,
1148 + "instant": false,
1149 + "interval": "",
1150 + "legendFormat": "__auto",
1151 + "range": true,
1152 + "refId": "A",
1153 + "useBackend": false
1154 + }
1155 + ],
1156 + "title": "Finish Reason",
1157 + "description": "Number of finished requests by their finish reason: either an EOS token was generated or the max sequence length was reached.",
1158 + "type": "timeseries"
1159 }
1160 ],
1161 "refresh": "",
requirements-common.txt
+1
@@ -12,6 +12,7 @@ openai
12 uvicorn[standard]
13 pydantic >= 2.0 # Required for OpenAI server.
14 prometheus_client >= 0.18.0
15 +prometheus-fastapi-instrumentator >= 7.0.0
16 tiktoken == 0.6.0 # Required for DBRX tokenizer
17 lm-format-enforcer == 0.9.8
18 outlines == 0.0.34 # Requires torch >= 2.1.0
vllm/core/scheduler.py
+1 -1
@@ -320,7 +320,7 @@ class Scheduler:
320 for seq_group in state_queue:
321 if not request_ids:
322 # Using 'break' here may add two extra iterations,
323 - # but is acceptable to reduce complexity .
323 + # but is acceptable to reduce complexity.
324 break
325 if seq_group.request_id in request_ids:
326 # Appending aborted group into pending list.
vllm/engine/llm_engine.py
+119 -52
@@ -22,7 +22,8 @@ from vllm.lora.request import LoRARequest
22 from vllm.outputs import RequestOutput
23 from vllm.sampling_params import SamplingParams
24 from vllm.sequence import (MultiModalData, SamplerOutput, Sequence,
25 - SequenceGroup, SequenceGroupMetadata)
25 + SequenceGroup, SequenceGroupMetadata,
26 + SequenceStatus)
27 from vllm.transformers_utils.detokenizer import Detokenizer
28 from vllm.transformers_utils.tokenizer_group import (BaseTokenizerGroup,
29 get_tokenizer_group)
@@ -217,7 +218,8 @@ class LLMEngine:
218 if self.log_stats:
219 self.stat_logger = StatLogger(
220 local_interval=_LOCAL_LOGGING_INTERVAL_SEC,
220 - labels=dict(model_name=model_config.model))
221 + labels=dict(model_name=model_config.model),
222 + max_model_len=self.model_config.max_model_len)
223 self.stat_logger.info("cache_config", self.cache_config)
224
225 # Create sequence output processor, e.g. for beam search or
@@ -619,59 +621,109 @@ class LLMEngine:
621 """
622 now = time.time()
623
622 - # KV Cache Usage in %.
624 + # System State
625 + # Scheduler State
626 + num_running_sys = len(self.scheduler.running)
627 + num_swapped_sys = len(self.scheduler.swapped)
628 + num_waiting_sys = len(self.scheduler.waiting)
629 +
630 + # KV Cache Usage in %
631 num_total_gpu = self.cache_config.num_gpu_blocks
632 num_free_gpu = self.scheduler.block_manager.get_num_free_gpu_blocks()
625 - gpu_cache_usage = 1.0 - (num_free_gpu / num_total_gpu)
633 + gpu_cache_usage_sys = 1.0 - (num_free_gpu / num_total_gpu)
634
635 num_total_cpu = self.cache_config.num_cpu_blocks
628 - cpu_cache_usage = 0.
636 + cpu_cache_usage_sys = 0.
637 if num_total_cpu > 0:
638 num_free_cpu = self.scheduler.block_manager.get_num_free_cpu_blocks(
639 )
632 - cpu_cache_usage = 1.0 - (num_free_cpu / num_total_cpu)
633 -
634 - # Scheduler State
635 - num_running = len(self.scheduler.running)
636 - num_swapped = len(self.scheduler.swapped)
637 - num_waiting = len(self.scheduler.waiting)
638 -
639 - # Iteration stats if we have scheduler output.
640 - num_prompt_tokens = 0
641 - num_generation_tokens = 0
642 - time_to_first_tokens = []
643 - time_per_output_tokens = []
644 - time_e2e_requests = []
640 + cpu_cache_usage_sys = 1.0 - (num_free_cpu / num_total_cpu)
641 +
642 + # Iteration stats
643 + num_prompt_tokens_iter = 0
644 + num_generation_tokens_iter = 0
645 + time_to_first_tokens_iter: List[float] = []
646 + time_per_output_tokens_iter: List[float] = []
647 +
648 + # Request stats
649 + # Latency
650 + time_e2e_requests: List[float] = []
651 + # Metadata
652 + num_prompt_tokens_requests: List[int] = []
653 + num_generation_tokens_requests: List[int] = []
654 + best_of_requests: List[int] = []
655 + n_requests: List[int] = []
656 + finished_reason_requests: List[str] = []
657 +
658 + # NOTE: This loop assumes prefill seq_groups are before
659 + # decode seq_groups in scheduled_seq_groups.
660 if scheduler_outputs is not None:
646 - prompt_run = scheduler_outputs.num_prefill_groups > 0
647 -
648 - # Number of Tokens.
649 - if prompt_run:
650 - num_prompt_tokens = sum(
651 - len(scheduled_seq_group.seq_group.prompt_token_ids)
652 - for scheduled_seq_group in
653 - scheduler_outputs.scheduled_seq_groups)
654 - num_generation_tokens = sum(
655 - scheduled_seq_group.seq_group.num_seqs()
656 - for scheduled_seq_group in
657 - scheduler_outputs.scheduled_seq_groups)
658 - else:
659 - num_generation_tokens = scheduler_outputs.num_batched_tokens
660 -
661 - # Latency Timings.
662 - time_last_iters = []
663 - for scheduled_seq_group in scheduler_outputs.scheduled_seq_groups:
661 + num_generation_tokens_from_prefill_groups = 0.
662 + if scheduler_outputs.num_prefill_groups > 0 and len(
663 + scheduler_outputs.scheduled_seq_groups
664 + ) != scheduler_outputs.num_prefill_groups:
665 + print("DETECTED CHUNKED")
666 +
667 + for idx, scheduled_seq_group in enumerate(
668 + scheduler_outputs.scheduled_seq_groups):
669 + group_was_prefill = idx < scheduler_outputs.num_prefill_groups
670 seq_group = scheduled_seq_group.seq_group
665 - # Time since last token.
666 - # (n.b. updates seq_group.metrics.last_token_time)
667 - time_last_iters.append(seq_group.get_last_latency(now))
668 - # Time since arrival for all finished requests.
671 +
672 + # NOTE: a seq_group that completed all of its prefill tokens
673 + # in the last iteration will have seq_group.is_prefill() = False
674 + # with group_was_prefill = True
675 + if group_was_prefill:
676 + # Number of prompt tokens.
677 + num_prompt_tokens_iter += (
678 + scheduled_seq_group.token_chunk_size)
679 +
680 + # If the seq_group just finished the prefill state
681 + # get TTFT.
682 + if not seq_group.is_prefill():
683 + latency = seq_group.get_last_latency(now)
684 + time_to_first_tokens_iter.append(latency)
685 +
686 + # One generation token per finished prefill.
687 + num_generation_tokens_from_prefill_groups += (
688 + seq_group.num_seqs())
689 + else:
690 + # TPOTs.
691 + latency = seq_group.get_last_latency(now)
692 + time_per_output_tokens_iter.append(latency)
693 +
694 + # Because of chunked prefill, we can have a single sequence
695 + # group that does multiple prompt_runs. To prevent logging
696 + # the same metadata more than once per request, we standardize
697 + # on logging request level information for finished requests,
698 + # which can only happen once.
699 if seq_group.is_finished():
700 + # Latency timings
701 time_e2e_requests.append(now -
702 seq_group.metrics.arrival_time)
703
673 - time_to_first_tokens = time_last_iters if prompt_run else []
674 - time_per_output_tokens = [] if prompt_run else time_last_iters
704 + # Metadata
705 + num_prompt_tokens_requests.append(
706 + len(seq_group.prompt_token_ids))
707 + num_generation_tokens_requests.extend([
708 + seq.get_output_len()
709 + for seq in seq_group.get_finished_seqs()
710 + ])
711 + best_of_requests.append(seq_group.sampling_params.best_of)
712 + n_requests.append(seq_group.sampling_params.n)
713 + finished_reason_requests.extend([
714 + SequenceStatus.get_finished_reason(seq.status)
715 + for seq in seq_group.get_finished_seqs()
716 + ])
717 +
718 + # Number of generation tokens.
719 + # num_batched_tokens equals the number of prompt_tokens plus the
720 + # number of decode_tokens in a single iteration. So,
721 + # num_generation_tokens = num_batched_tokens - num_prompt_tokens
722 + # + num_generation_tokens_from_prefill_groups (since we generate
723 + # one token on prefills on iters where the prefill finishes).
724 + num_generation_tokens_iter = (
725 + scheduler_outputs.num_batched_tokens - num_prompt_tokens_iter +
726 + num_generation_tokens_from_prefill_groups)
727
728 # Spec decode, if enabled, emits specialized metrics from the worker in
729 # sampler output.
@@ -683,17 +735,32 @@ class LLMEngine:
735
736 return Stats(
737 now=now,
686 - num_running=num_running,
687 - num_swapped=num_swapped,
688 - num_waiting=num_waiting,
689 - gpu_cache_usage=gpu_cache_usage,
690 - cpu_cache_usage=cpu_cache_usage,
691 - num_prompt_tokens=num_prompt_tokens,
692 - num_generation_tokens=num_generation_tokens,
693 - time_to_first_tokens=time_to_first_tokens,
694 - time_per_output_tokens=time_per_output_tokens,
695 - time_e2e_requests=time_e2e_requests,
738 +
739 + # System stats
740 + # Scheduler State
741 + num_running_sys=num_running_sys,
742 + num_swapped_sys=num_swapped_sys,
743 + num_waiting_sys=num_waiting_sys,
744 + # KV Cache Usage in %
745 + gpu_cache_usage_sys=gpu_cache_usage_sys,
746 + cpu_cache_usage_sys=cpu_cache_usage_sys,
747 +
748 + # Iteration stats
749 + num_prompt_tokens_iter=num_prompt_tokens_iter,
750 + num_generation_tokens_iter=num_generation_tokens_iter,
751 + time_to_first_tokens_iter=time_to_first_tokens_iter,
752 + time_per_output_tokens_iter=time_per_output_tokens_iter,
753 spec_decode_metrics=spec_decode_metrics,
754 +
755 + # Request stats
756 + # Latency
757 + time_e2e_requests=time_e2e_requests,
758 + # Metadata
759 + num_prompt_tokens_requests=num_prompt_tokens_requests,
760 + num_generation_tokens_requests=num_generation_tokens_requests,
761 + best_of_requests=best_of_requests,
762 + n_requests=n_requests,
763 + finished_reason_requests=finished_reason_requests,
764 )
765
766 def add_lora(self, lora_request: LoRARequest) -> bool:
vllm/engine/metrics.py
+163 -58
@@ -1,6 +1,8 @@
1 import time
2 from dataclasses import dataclass
3 -from typing import TYPE_CHECKING, Dict, List, Optional, Protocol
3 +from typing import TYPE_CHECKING
4 +from typing import Counter as CollectionsCounter
5 +from typing import Dict, List, Optional, Protocol, Union
6
7 import numpy as np
8 from prometheus_client import (REGISTRY, Counter, Gauge, Histogram, Info,
@@ -21,8 +23,9 @@ disable_created_metrics()
23
24 # begin-metrics-definitions
25 class Metrics:
26 + labelname_finish_reason = "finished_reason"
27
25 - def __init__(self, labelnames: List[str]):
28 + def __init__(self, labelnames: List[str], max_model_len: int):
29 # Unregister any existing vLLM collectors
30 for collector in list(REGISTRY._collector_to_names):
31 if hasattr(collector, "_name") and "vllm" in collector._name:
@@ -34,18 +37,20 @@ class Metrics:
37 documentation='information of cache_config')
38
39 # System stats
40 + # Scheduler State
41 self.gauge_scheduler_running = Gauge(
42 name="vllm:num_requests_running",
43 documentation="Number of requests currently running on GPU.",
44 labelnames=labelnames)
41 - self.gauge_scheduler_swapped = Gauge(
42 - name="vllm:num_requests_swapped",
43 - documentation="Number of requests swapped to CPU.",
44 - labelnames=labelnames)
45 self.gauge_scheduler_waiting = Gauge(
46 name="vllm:num_requests_waiting",
47 documentation="Number of requests waiting to be processed.",
48 labelnames=labelnames)
49 + self.gauge_scheduler_swapped = Gauge(
50 + name="vllm:num_requests_swapped",
51 + documentation="Number of requests swapped to CPU.",
52 + labelnames=labelnames)
53 + # KV Cache Usage in %
54 self.gauge_gpu_cache_usage = Gauge(
55 name="vllm:gpu_cache_usage_perc",
56 documentation="GPU KV-cache usage. 1 means 100 percent usage.",
@@ -55,7 +60,7 @@ class Metrics:
60 documentation="CPU KV-cache usage. 1 means 100 percent usage.",
61 labelnames=labelnames)
62
58 - # Raw stats from last model iteration
63 + # Iteration stats
64 self.counter_prompt_tokens = Counter(
65 name="vllm:prompt_tokens_total",
66 documentation="Number of prefill tokens processed.",
@@ -80,18 +85,51 @@ class Metrics:
85 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
86 1.0, 2.5
87 ])
83 - self.histogram_e2e_request_latency = Histogram(
88 +
89 + # Request stats
90 + # Latency
91 + self.histogram_e2e_time_request = Histogram(
92 name="vllm:e2e_request_latency_seconds",
93 documentation="Histogram of end to end request latency in seconds.",
94 labelnames=labelnames,
95 buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
96 + # Metadata
97 + self.histogram_num_prompt_tokens_request = Histogram(
98 + name="vllm:request_prompt_tokens",
99 + documentation="Number of prefill tokens processed.",
100 + labelnames=labelnames,
101 + buckets=build_1_2_5_buckets(max_model_len),
102 + )
103 + self.histogram_num_generation_tokens_request = Histogram(
104 + name="vllm:request_generation_tokens",
105 + documentation="Number of generation tokens processed.",
106 + labelnames=labelnames,
107 + buckets=build_1_2_5_buckets(max_model_len),
108 + )
109 + self.histogram_best_of_request = Histogram(
110 + name="vllm:request_params_best_of",
111 + documentation="Histogram of the best_of request parameter.",
112 + labelnames=labelnames,
113 + buckets=[1, 2, 5, 10, 20],
114 + )
115 + self.histogram_n_request = Histogram(
116 + name="vllm:request_params_n",
117 + documentation="Histogram of the n request parameter.",
118 + labelnames=labelnames,
119 + buckets=[1, 2, 5, 10, 20],
120 + )
121 + self.counter_request_success = Counter(
122 + name="vllm:request_success",
123 + documentation="Count of successfully processed requests.",
124 + labelnames=labelnames + [Metrics.labelname_finish_reason])
125
89 - # Legacy metrics
126 + # Deprecated in favor of vllm:prompt_tokens_total
127 self.gauge_avg_prompt_throughput = Gauge(
128 name="vllm:avg_prompt_throughput_toks_per_s",
129 documentation="Average prefill throughput in tokens/s.",
130 labelnames=labelnames,
131 )
132 + # Deprecated in favor of vllm:generation_tokens_total
133 self.gauge_avg_generation_throughput = Gauge(
134 name="vllm:avg_generation_throughput_toks_per_s",
135 documentation="Average generation throughput in tokens/s.",
@@ -102,24 +140,57 @@ class Metrics:
140 # end-metrics-definitions
141
142
143 +def build_1_2_5_buckets(max_value: int):
144 + """
145 + Builds a list of buckets with increasing powers of 10 multiplied by
146 + mantissa values (1, 2, 5) until the value exceeds the specified maximum.
147 +
148 + Example:
149 + >>> build_1_2_5_buckets(100)
150 + [1, 2, 5, 10, 20, 50, 100]
151 + """
152 + mantissa_lst = [1, 2, 5]
153 + exponent = 0
154 + buckets = []
155 + while True:
156 + for m in mantissa_lst:
157 + value = m * 10**exponent
158 + if value <= max_value:
159 + buckets.append(value)
160 + else:
161 + return buckets
162 + exponent += 1
163 +
164 +
165 @dataclass
166 class Stats:
167 """Created by LLMEngine for use by StatLogger."""
168 now: float
169
110 - # System stats.
111 - num_running: int
112 - num_waiting: int
113 - num_swapped: int
114 - gpu_cache_usage: float
115 - cpu_cache_usage: float
116 -
117 - # Raw stats from last model iteration.
118 - num_prompt_tokens: int
119 - num_generation_tokens: int
120 - time_to_first_tokens: List[float]
121 - time_per_output_tokens: List[float]
170 + # System stats (should have _sys suffix)
171 + # Scheduler State
172 + num_running_sys: int
173 + num_waiting_sys: int
174 + num_swapped_sys: int
175 + # KV Cache Usage in %
176 + gpu_cache_usage_sys: float
177 + cpu_cache_usage_sys: float
178 +
179 + # Iteration stats (should have _iter suffix)
180 + num_prompt_tokens_iter: int
181 + num_generation_tokens_iter: int
182 + time_to_first_tokens_iter: List[float]
183 + time_per_output_tokens_iter: List[float]
184 +
185 + # Request stats (should have _requests suffix)
186 + # Latency
187 time_e2e_requests: List[float]
188 + # Metadata
189 + num_prompt_tokens_requests: List[int]
190 + num_generation_tokens_requests: List[int]
191 + best_of_requests: List[int]
192 + n_requests: List[int]
193 + finished_reason_requests: List[str]
194
195 spec_decode_metrics: Optional["SpecDecodeWorkerMetrics"] = None
196
@@ -133,7 +204,8 @@ class SupportsMetricsInfo(Protocol):
204 class StatLogger:
205 """StatLogger is used LLMEngine to log to Promethus and Stdout."""
206
136 - def __init__(self, local_interval: float, labels: Dict[str, str]) -> None:
207 + def __init__(self, local_interval: float, labels: Dict[str, str],
208 + max_model_len: int) -> None:
209 # Metadata for logging locally.
210 self.last_local_log = time.time()
211 self.local_interval = local_interval
@@ -144,7 +216,8 @@ class StatLogger:
216
217 # Prometheus metrics
218 self.labels = labels
147 - self.metrics = Metrics(labelnames=list(labels.keys()))
219 + self.metrics = Metrics(labelnames=list(labels.keys()),
220 + max_model_len=max_model_len)
221
222 def info(self, type: str, obj: SupportsMetricsInfo) -> None:
223 if type == "cache_config":
@@ -158,34 +231,66 @@ class StatLogger:
231 return elapsed_time > self.local_interval
232
233 def _log_prometheus(self, stats: Stats) -> None:
161 - # Set system stat gauges.
162 - self.metrics.gauge_scheduler_running.labels(**self.labels).set(
163 - stats.num_running)
164 - self.metrics.gauge_scheduler_swapped.labels(**self.labels).set(
165 - stats.num_swapped)
166 - self.metrics.gauge_scheduler_waiting.labels(**self.labels).set(
167 - stats.num_waiting)
168 - self.metrics.gauge_gpu_cache_usage.labels(**self.labels).set(
169 - stats.gpu_cache_usage)
170 - self.metrics.gauge_cpu_cache_usage.labels(**self.labels).set(
171 - stats.cpu_cache_usage)
172 -
173 - # Add to token counters.
174 - self.metrics.counter_prompt_tokens.labels(**self.labels).inc(
175 - stats.num_prompt_tokens)
176 - self.metrics.counter_generation_tokens.labels(**self.labels).inc(
177 - stats.num_generation_tokens)
178 -
179 - # Observe request level latencies in histograms.
180 - for ttft in stats.time_to_first_tokens:
181 - self.metrics.histogram_time_to_first_token.labels(
182 - **self.labels).observe(ttft)
183 - for tpot in stats.time_per_output_tokens:
184 - self.metrics.histogram_time_per_output_token.labels(
185 - **self.labels).observe(tpot)
186 - for e2e in stats.time_e2e_requests:
187 - self.metrics.histogram_e2e_request_latency.labels(
188 - **self.labels).observe(e2e)
234 + # System state data
235 + self._log_gauge(self.metrics.gauge_scheduler_running,
236 + stats.num_running_sys)
237 + self._log_gauge(self.metrics.gauge_scheduler_swapped,
238 + stats.num_swapped_sys)
239 + self._log_gauge(self.metrics.gauge_scheduler_waiting,
240 + stats.num_waiting_sys)
241 + self._log_gauge(self.metrics.gauge_gpu_cache_usage,
242 + stats.gpu_cache_usage_sys)
243 + self._log_gauge(self.metrics.gauge_cpu_cache_usage,
244 + stats.cpu_cache_usage_sys)
245 +
246 + # Iteration level data
247 + self._log_counter(self.metrics.counter_prompt_tokens,
248 + stats.num_prompt_tokens_iter)
249 + self._log_counter(self.metrics.counter_generation_tokens,
250 + stats.num_generation_tokens_iter)
251 + self._log_histogram(self.metrics.histogram_time_to_first_token,
252 + stats.time_to_first_tokens_iter)
253 + self._log_histogram(self.metrics.histogram_time_per_output_token,
254 + stats.time_per_output_tokens_iter)
255 +
256 + # Request level data
257 + # Latency
258 + self._log_histogram(self.metrics.histogram_e2e_time_request,
259 + stats.time_e2e_requests)
260 + # Metadata
261 + finished_reason_counter = CollectionsCounter(
262 + stats.finished_reason_requests)
263 + self._log_counter_labels(self.metrics.counter_request_success,
264 + finished_reason_counter,
265 + Metrics.labelname_finish_reason)
266 + self._log_histogram(self.metrics.histogram_num_prompt_tokens_request,
267 + stats.num_prompt_tokens_requests)
268 + self._log_histogram(
269 + self.metrics.histogram_num_generation_tokens_request,
270 + stats.num_generation_tokens_requests)
271 + self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
272 + self._log_histogram(self.metrics.histogram_best_of_request,
273 + stats.best_of_requests)
274 +
275 + def _log_gauge(self, gauge: Gauge, data: Union[int, float]) -> None:
276 + # Convenience function for logging to gauge.
277 + gauge.labels(**self.labels).set(data)
278 +
279 + def _log_counter(self, counter: Counter, data: Union[int, float]) -> None:
280 + # Convenience function for logging to counter.
281 + counter.labels(**self.labels).inc(data)
282 +
283 + def _log_counter_labels(self, counter: Counter, data: CollectionsCounter,
284 + label_key: str) -> None:
285 + # Convenience function for collection counter of labels.
286 + for label, count in data.items():
287 + counter.labels(**{**self.labels, label_key: label}).inc(count)
288 +
289 + def _log_histogram(self, histogram: Histogram,
290 + data: Union[List[int], List[float]]) -> None:
291 + # Convenience function for logging list to histogram.
292 + for datum in data:
293 + histogram.labels(**self.labels).observe(datum)
294
295 def _log_prometheus_interval(self, prompt_throughput: float,
296 generation_throughput: float) -> None:
@@ -210,8 +315,8 @@ class StatLogger:
315 self._log_prometheus(stats)
316
317 # Save tracked stats for token counters.
213 - self.num_prompt_tokens.append(stats.num_prompt_tokens)
214 - self.num_generation_tokens.append(stats.num_generation_tokens)
318 + self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
319 + self.num_generation_tokens.append(stats.num_generation_tokens_iter)
320
321 # Log locally every local_interval seconds.
322 if self._local_interval_elapsed(stats.now):
@@ -234,11 +339,11 @@ class StatLogger:
339 "CPU KV cache usage: %.1f%%",
340 prompt_throughput,
341 generation_throughput,
237 - stats.num_running,
238 - stats.num_swapped,
239 - stats.num_waiting,
240 - stats.gpu_cache_usage * 100,
241 - stats.cpu_cache_usage * 100,
342 + stats.num_running_sys,
343 + stats.num_swapped_sys,
344 + stats.num_waiting_sys,
345 + stats.gpu_cache_usage_sys * 100,
346 + stats.cpu_cache_usage_sys * 100,
347 )
348
349 # Reset tracked stats for next interval.
vllm/sequence.py
+15 -3
@@ -442,15 +442,27 @@ class SequenceGroup:
442 def lora_int_id(self) -> int:
443 return self.lora_request.lora_int_id if self.lora_request else 0
444
445 - def get_last_latency(self, now: float) -> float:
446 - """Gets last token latency for Request level timings."""
445 + def get_last_latency(self, now: float) -> Optional[float]:
446 + """Sets the last token time for Request level timings."""
447 + # If still in prefill phase, raise Error.
448 + if self.is_prefill():
449 + raise ValueError(
450 + "seq_group.get_last_latency() should not be called "
451 + "if the seq_group is in prefill phase.")
452 +
453 + # Otherwise return token latency.
454 latency = now - self.metrics.last_token_time
455 self.metrics.last_token_time = now
456 return latency
457
458 def maybe_set_first_token_time(self, time: float) -> None:
459 """Sets the first token time for Request level timings."""
453 - if self.metrics.first_token_time is None:
460 + # Note: in a case where a sequence_group is swapped and
461 + # recomputed, the time between iterations is counted
462 + # in TPOT, rather than recalculating TTFT (since from the )
463 + # POV of the user, there is simply a long generation delay.
464 + if (self.metrics.first_token_time is None
465 + and self.get_seqs()[0].get_output_len() == 1):
466 self.metrics.first_token_time = time
467
468 def maybe_set_first_scheduled_time(self, time: float) -> None: