| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from dataclasses import dataclass |
| 5 | |
| 6 | from prometheus_client import REGISTRY |
| 7 | from prometheus_client import Metric as PromMetric |
| 8 | from prometheus_client.samples import Sample |
| 9 | |
| 10 | |
| 11 | @dataclass |
| 12 | class Metric: |
| 13 | """A base class for prometheus metrics. |
| 14 | |
| 15 | Each metric may be associated with key=value labels, and |
| 16 | in some cases a single vLLM instance may have multiple |
| 17 | metrics with the same name but different sets of labels. |
| 18 | """ |
| 19 | |
| 20 | name: str |
| 21 | labels: dict[str, str] |
| 22 | |
| 23 | |
| 24 | @dataclass |
| 25 | class Counter(Metric): |
| 26 | """A monotonically increasing integer counter.""" |
| 27 | |
| 28 | value: int |
| 29 | |
| 30 | |
| 31 | @dataclass |
| 32 | class Vector(Metric): |
| 33 | """An ordered array of integer counters. |
| 34 | |
| 35 | This type - which doesn't exist in Prometheus - models one very |
| 36 | specific metric, vllm:spec_decode_num_accepted_tokens_per_pos. |
| 37 | """ |
| 38 | |
| 39 | values: list[int] |
| 40 | |
| 41 | |
| 42 | @dataclass |
| 43 | class Gauge(Metric): |
| 44 | """A numerical value that can go up or down.""" |
| 45 | |
| 46 | value: float |
| 47 | |
| 48 | |
| 49 | @dataclass |
| 50 | class Histogram(Metric): |
| 51 | """Observations recorded in configurable buckets. |
| 52 | |
| 53 | Buckets are represented by a dictionary. The key is |
| 54 | the upper limit of the bucket, and the value is the |
| 55 | observed count in that bucket. A '+Inf' key always |
| 56 | exists. |
| 57 | |
| 58 | The count property is the total count across all |
| 59 | buckets, identical to the count of the '+Inf' bucket. |
| 60 | |
| 61 | The sum property is the total sum of all observed |
| 62 | values. |
| 63 | """ |
| 64 | |
| 65 | count: int |
| 66 | sum: float |
| 67 | buckets: dict[str, int] |
| 68 | |
| 69 | |
| 70 | def get_metrics_snapshot() -> list[Metric]: |
| 71 | """An API for accessing in-memory Prometheus metrics. |
| 72 | |
| 73 | Example: |
| 74 | >>> for metric in llm.get_metrics(): |
| 75 | ... if isinstance(metric, Counter): |
| 76 | ... print(f"{metric} = {metric.value}") |
| 77 | ... elif isinstance(metric, Gauge): |
| 78 | ... print(f"{metric} = {metric.value}") |
| 79 | ... elif isinstance(metric, Histogram): |
| 80 | ... print(f"{metric}") |
| 81 | ... print(f" sum = {metric.sum}") |
| 82 | ... print(f" count = {metric.count}") |
| 83 | ... for bucket_le, value in metrics.buckets.items(): |
| 84 | ... print(f" {bucket_le} = {value}") |
| 85 | |
| 86 | """ |
| 87 | collected: list[Metric] = [] |
| 88 | for metric in REGISTRY.collect(): |
| 89 | if not metric.name.startswith("vllm:"): |
| 90 | continue |
| 91 | if metric.type == "gauge": |
| 92 | samples = _get_samples(metric) |
| 93 | for s in samples: |
| 94 | collected.append( |
| 95 | Gauge(name=metric.name, labels=s.labels, value=s.value) |
| 96 | ) |
| 97 | elif metric.type == "counter": |
| 98 | samples = _get_samples(metric, "_total") |
| 99 | if metric.name == "vllm:spec_decode_num_accepted_tokens_per_pos": |
| 100 | # |
| 101 | # Ugly vllm:num_accepted_tokens_per_pos special case. |
| 102 | # |
| 103 | # This metric is a vector of counters - for each spec |
| 104 | # decoding token position, we observe the number of |
| 105 | # accepted tokens using a Counter labeled with 'position'. |
| 106 | # We convert these into a vector of integer values. |
| 107 | # |
| 108 | for labels, values in _digest_num_accepted_by_pos_samples(samples): |
| 109 | collected.append( |
| 110 | Vector(name=metric.name, labels=labels, values=values) |
| 111 | ) |
| 112 | else: |
| 113 | for s in samples: |
| 114 | collected.append( |
| 115 | Counter(name=metric.name, labels=s.labels, value=int(s.value)) |
| 116 | ) |
| 117 | |
| 118 | elif metric.type == "histogram": |
| 119 | # |
| 120 | # A histogram has a number of '_bucket' samples where |
| 121 | # the 'le' label represents the upper limit of the bucket. |
| 122 | # We convert these bucketized values into a dict of values |
| 123 | # indexed by the value of the 'le' label. The 'le=+Inf' |
| 124 | # label is a special case, catching all values observed. |
| 125 | # |
| 126 | bucket_samples = _get_samples(metric, "_bucket") |
| 127 | count_samples = _get_samples(metric, "_count") |
| 128 | sum_samples = _get_samples(metric, "_sum") |
| 129 | for labels, buckets, count_value, sum_value in _digest_histogram( |
| 130 | bucket_samples, count_samples, sum_samples |
| 131 | ): |
| 132 | collected.append( |
| 133 | Histogram( |
| 134 | name=metric.name, |
| 135 | labels=labels, |
| 136 | buckets=buckets, |
| 137 | count=count_value, |
| 138 | sum=sum_value, |
| 139 | ) |
| 140 | ) |
| 141 | else: |
| 142 | raise AssertionError(f"Unknown metric type {metric.type}") |
| 143 | |
| 144 | return collected |
| 145 | |
| 146 | |
| 147 | def _get_samples(metric: PromMetric, suffix: str | None = None) -> list[Sample]: |
| 148 | name = (metric.name + suffix) if suffix is not None else metric.name |
| 149 | return [s for s in metric.samples if s.name == name] |
| 150 | |
| 151 | |
| 152 | def _strip_label(labels: dict[str, str], key_to_remove: str) -> dict[str, str]: |
| 153 | labels_copy = labels.copy() |
| 154 | labels_copy.pop(key_to_remove) |
| 155 | return labels_copy |
| 156 | |
| 157 | |
| 158 | def _digest_histogram( |
| 159 | bucket_samples: list[Sample], count_samples: list[Sample], sum_samples: list[Sample] |
| 160 | ) -> list[tuple[dict[str, str], dict[str, int], int, float]]: |
| 161 | # |
| 162 | # In the case of DP, we have an indigestable |
| 163 | # per-bucket-per-engine count as a list of labelled |
| 164 | # samples, along with total and sum samples |
| 165 | # |
| 166 | # bucket_samples (in): |
| 167 | # labels = {bucket: 100, idx: 0}, value = 2 |
| 168 | # labels = {bucket: 200, idx: 0}, value = 4 |
| 169 | # labels = {bucket: Inf, idx: 0}, value = 10 |
| 170 | # labels = {bucket: 100, idx: 1}, value = 1 |
| 171 | # labels = {bucket: 200, idx: 2}, value = 5 |
| 172 | # labels = {bucket: Inf, idx: 3}, value = 7 |
| 173 | # count_samples (in): |
| 174 | # labels = {idx: 0}, value = 10 |
| 175 | # labels = {idx: 1}, value = 7 |
| 176 | # sum_samples (in): |
| 177 | # labels = {idx: 0}, value = 2000 |
| 178 | # labels = {idx: 1}, value = 1200 |
| 179 | # |
| 180 | # output: [ |
| 181 | # {idx: 0}, {"100": 2, "200": 4, "Inf": 10}, 10, 2000 |
| 182 | # {idx: 1}, {"100": 1, "200": 5, "Inf": 7}, 7, 1200 |
| 183 | # ] |
| 184 | buckets_by_labels: dict[frozenset[tuple[str, str]], dict[str, int]] = {} |
| 185 | for s in bucket_samples: |
| 186 | bucket = s.labels["le"] |
| 187 | labels_key = frozenset(_strip_label(s.labels, "le").items()) |
| 188 | if labels_key not in buckets_by_labels: |
| 189 | buckets_by_labels[labels_key] = {} |
| 190 | buckets_by_labels[labels_key][bucket] = int(s.value) |
| 191 | |
| 192 | counts_by_labels: dict[frozenset[tuple[str, str]], int] = {} |
| 193 | for s in count_samples: |
| 194 | labels_key = frozenset(s.labels.items()) |
| 195 | counts_by_labels[labels_key] = int(s.value) |
| 196 | |
| 197 | sums_by_labels: dict[frozenset[tuple[str, str]], float] = {} |
| 198 | for s in sum_samples: |
| 199 | labels_key = frozenset(s.labels.items()) |
| 200 | sums_by_labels[labels_key] = s.value |
| 201 | |
| 202 | assert ( |
| 203 | set(buckets_by_labels.keys()) |
| 204 | == set(counts_by_labels.keys()) |
| 205 | == set(sums_by_labels.keys()) |
| 206 | ) |
| 207 | |
| 208 | output = [] |
| 209 | label_keys = list(buckets_by_labels.keys()) |
| 210 | for k in label_keys: |
| 211 | labels = dict(k) |
| 212 | output.append( |
| 213 | (labels, buckets_by_labels[k], counts_by_labels[k], sums_by_labels[k]) |
| 214 | ) |
| 215 | return output |
| 216 | |
| 217 | |
| 218 | def _digest_num_accepted_by_pos_samples( |
| 219 | samples: list[Sample], |
| 220 | ) -> list[tuple[dict[str, str], list[int]]]: |
| 221 | # |
| 222 | # In the case of DP, we have an indigestable |
| 223 | # per-position-per-engine count as a list of |
| 224 | # labelled samples |
| 225 | # |
| 226 | # samples (in): |
| 227 | # labels = {pos: 0, idx: 0}, value = 10 |
| 228 | # labels = {pos: 1, idx: 0}, value = 7 |
| 229 | # labels = {pos: 2, idx: 0}, value = 2 |
| 230 | # labels = {pos: 0, idx: 1}, value = 5 |
| 231 | # labels = {pos: 1, idx: 1}, value = 3 |
| 232 | # labels = {pos: 2, idx: 1}, value = 1 |
| 233 | # |
| 234 | # output: [ |
| 235 | # {idx: 0}, [10, 7, 2] |
| 236 | # {idx: 1}, [5, 3, 1] |
| 237 | # ] |
| 238 | # |
| 239 | max_pos = 0 |
| 240 | values_by_labels: dict[frozenset[tuple[str, str]], dict[int, int]] = {} |
| 241 | |
| 242 | for s in samples: |
| 243 | position = int(s.labels["position"]) |
| 244 | max_pos = max(max_pos, position) |
| 245 | |
| 246 | labels_key = frozenset(_strip_label(s.labels, "position").items()) |
| 247 | if labels_key not in values_by_labels: |
| 248 | values_by_labels[labels_key] = {} |
| 249 | values_by_labels[labels_key][position] = int(s.value) |
| 250 | |
| 251 | output = [] |
| 252 | for labels_key, values_by_position in values_by_labels.items(): |
| 253 | labels = dict(labels_key) |
| 254 | values = [0] * (max_pos + 1) |
| 255 | for pos, val in values_by_position.items(): |
| 256 | values[pos] = val |
| 257 | output.append((labels, values)) |
| 258 | return output |