A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[V1][Metrics] Add API for accessing in-memory Prometheus metrics (#17010)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
May 27, 2025 at 10:37 UTC
06a0338015a3ac959edb4d152a69793359983a4d
10 files changed
+543
-28
.buildkite/test-pipeline.yaml
+1
@@ -222,6 +222,7 @@ steps:
222
- pytest -v -s v1/test_serial_utils.py
223
- pytest -v -s v1/test_utils.py
224
- pytest -v -s v1/test_oracle.py
225
+ - pytest -v -s v1/test_metrics_reader.py
226
# TODO: accuracy does not match, whether setting
227
# VLLM_USE_FLASHINFER_SAMPLER or not on H100.
228
- pytest -v -s v1/e2e
examples/offline_inference/eagle.py
+20
-16
@@ -6,6 +6,7 @@ import os
6
from transformers import AutoTokenizer
7
8
from vllm import LLM, SamplingParams
9
+from vllm.v1.metrics.reader import Counter, Vector
10
11
12
def load_prompts(dataset_path, num_prompts):
@@ -105,30 +106,33 @@ def main():
106
print(f"generated text: {output.outputs[0].text}")
107
print("-" * 50)
108
108
- if not hasattr(outputs, "metrics") or outputs.metrics is None:
109
+ try:
110
+ metrics = llm.get_metrics()
111
+ except AssertionError:
112
+ print("Metrics are not supported in the V0 engine.")
113
return
114
111
- # calculate the average number of accepted tokens per forward pass, +1 is
112
- # to account for the token from the target model that's always going to be
113
- # accepted
114
- acceptance_counts = [0] * (args.num_spec_tokens + 1)
115
- for output in outputs:
116
- for step, count in enumerate(output.metrics.spec_token_acceptance_counts):
117
- acceptance_counts[step] += count
115
+ num_drafts = num_accepted = 0
116
+ acceptance_counts = [0] * args.num_spec_tokens
117
+ for metric in metrics:
118
+ if metric.name == "vllm:spec_decode_num_drafts":
119
+ assert isinstance(metric, Counter)
120
+ num_drafts += metric.value
121
+ elif metric.name == "vllm:spec_decode_num_accepted_tokens":
122
+ assert isinstance(metric, Counter)
123
+ num_accepted += metric.value
124
+ elif metric.name == "vllm:spec_decode_num_accepted_tokens_per_pos":
125
+ assert isinstance(metric, Vector)
126
+ for pos in range(len(metric.values)):
127
+ acceptance_counts[pos] += metric.values[pos]
128
129
print("-" * 50)
120
- print(
121
- f"mean acceptance length (including bonus tokens): \
122
- {1 + (sum(acceptance_counts) / acceptance_counts[0]):.2f}"
123
- )
130
+ print(f"mean acceptance length: {1 + (num_accepted / num_drafts):.2f}")
131
print("-" * 50)
132
133
# print acceptance at each token position
134
for i in range(len(acceptance_counts)):
128
- print(
129
- f"acceptance at token {i}:"
130
- f"{acceptance_counts[i] / (acceptance_counts[0]):.2f}"
131
- )
135
+ print(f"acceptance at token {i}:{acceptance_counts[i] / num_drafts:.2f}")
136
137
138
if __name__ == "__main__":
examples/offline_inference/metrics.py
new
+49
@@ -0,0 +1,49 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+from vllm import LLM, SamplingParams
4
+from vllm.v1.metrics.reader import Counter, Gauge, Histogram, Vector
5
+
6
+# Sample prompts.
7
+prompts = [
8
+ "Hello, my name is",
9
+ "The president of the United States is",
10
+ "The capital of France is",
11
+ "The future of AI is",
12
+]
13
+# Create a sampling params object.
14
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
15
+
16
+
17
+def main():
18
+ # Create an LLM.
19
+ llm = LLM(model="facebook/opt-125m", disable_log_stats=False)
20
+
21
+ # Generate texts from the prompts.
22
+ outputs = llm.generate(prompts, sampling_params)
23
+
24
+ # Print the outputs.
25
+ print("-" * 50)
26
+ for output in outputs:
27
+ prompt = output.prompt
28
+ generated_text = output.outputs[0].text
29
+ print(f"Prompt: {prompt!r}\nGenerated text: {generated_text!r}")
30
+ print("-" * 50)
31
+
32
+ # Dump all metrics
33
+ for metric in llm.get_metrics():
34
+ if isinstance(metric, Gauge):
35
+ print(f"{metric.name} (gauge) = {metric.value}")
36
+ elif isinstance(metric, Counter):
37
+ print(f"{metric.name} (counter) = {metric.value}")
38
+ elif isinstance(metric, Vector):
39
+ print(f"{metric.name} (vector) = {metric.values}")
40
+ elif isinstance(metric, Histogram):
41
+ print(f"{metric.name} (histogram)")
42
+ print(f" sum = {metric.sum}")
43
+ print(f" count = {metric.count}")
44
+ for bucket_le, value in metric.buckets.items():
45
+ print(f" {bucket_le} = {value}")
46
+
47
+
48
+if __name__ == "__main__":
49
+ main()
tests/v1/engine/test_llm_engine.py
+65
@@ -6,6 +6,7 @@ from typing import Optional
6
import pytest
7
8
from vllm import LLM, SamplingParams
9
+from vllm.v1.metrics.reader import Counter, Gauge, Histogram, Metric, Vector
10
11
MODEL = "facebook/opt-125m"
12
DTYPE = "half"
@@ -97,3 +98,67 @@ def test_parallel_sampling(vllm_model, example_prompts) -> None:
98
raise AssertionError(
99
f"{len(completion_counts)} unique completions; expected"
100
f" {n}. Repeats: {repeats}")
101
+
102
+
103
+def test_engine_metrics(vllm_runner, monkeypatch, example_prompts):
104
+ max_tokens = 100
105
+ # Use spec decoding to test num_accepted_tokens_per_pos
106
+ speculative_config = {
107
+ "method": "ngram",
108
+ "prompt_lookup_max": 5,
109
+ "prompt_lookup_min": 3,
110
+ "num_speculative_tokens": 5,
111
+ }
112
+ monkeypatch.setenv("VLLM_USE_V1", "1")
113
+ with vllm_runner(
114
+ MODEL,
115
+ speculative_config=speculative_config,
116
+ disable_log_stats=False,
117
+ ) as vllm_model:
118
+ model: LLM = vllm_model.model
119
+ sampling_params = SamplingParams(temperature=0.0,
120
+ max_tokens=max_tokens)
121
+ outputs = model.generate(example_prompts, sampling_params)
122
+
123
+ n_prompts = len(example_prompts)
124
+ assert len(outputs) == n_prompts
125
+
126
+ total_tokens = 0
127
+ for out in outputs:
128
+ assert len(out.outputs) == 1
129
+ total_tokens += len(out.outputs[0].token_ids)
130
+ assert total_tokens == max_tokens * n_prompts
131
+
132
+ metrics = model.get_metrics()
133
+
134
+ def find_metric(name) -> list[Metric]:
135
+ found = []
136
+ for metric in metrics:
137
+ if metric.name == name:
138
+ found.append(metric)
139
+ return found
140
+
141
+ num_requests_running = find_metric("vllm:num_requests_running")
142
+ assert len(num_requests_running) == 1
143
+ assert isinstance(num_requests_running[0], Gauge)
144
+ assert num_requests_running[0].value == .0
145
+
146
+ generation_tokens = find_metric("vllm:generation_tokens")
147
+ assert len(generation_tokens) == 1
148
+ assert isinstance(generation_tokens[0], Counter)
149
+ assert generation_tokens[0].value == total_tokens
150
+
151
+ request_generation_tokens = find_metric(
152
+ "vllm:request_generation_tokens")
153
+ assert len(request_generation_tokens) == 1
154
+ assert isinstance(request_generation_tokens[0], Histogram)
155
+ assert "+Inf" in request_generation_tokens[0].buckets
156
+ assert request_generation_tokens[0].buckets["+Inf"] == n_prompts
157
+ assert request_generation_tokens[0].count == n_prompts
158
+ assert request_generation_tokens[0].sum == total_tokens
159
+
160
+ num_accepted_tokens_per_pos = find_metric(
161
+ "vllm:spec_decode_num_accepted_tokens_per_pos")
162
+ assert len(num_accepted_tokens_per_pos) == 1
163
+ assert isinstance(num_accepted_tokens_per_pos[0], Vector)
164
+ assert len(num_accepted_tokens_per_pos[0].values) == 5
tests/v1/test_metrics_reader.py
new
+112
@@ -0,0 +1,112 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+import prometheus_client
4
+import pytest
5
+
6
+from vllm.v1.metrics.reader import (Counter, Gauge, Histogram, Vector,
7
+ get_metrics_snapshot)
8
+
9
+
10
+@pytest.fixture(autouse=True)
11
+def test_registry(monkeypatch):
12
+ # Use a custom registry for tests
13
+ test_registry = prometheus_client.CollectorRegistry(auto_describe=True)
14
+ monkeypatch.setattr("vllm.v1.metrics.reader.REGISTRY", test_registry)
15
+ return test_registry
16
+
17
+
18
+@pytest.mark.parametrize("num_engines", [1, 4])
19
+def test_gauge_metric(test_registry, num_engines):
20
+ g = prometheus_client.Gauge("vllm:test_gauge",
21
+ "Test gauge metric",
22
+ labelnames=["model", "engine_index"],
23
+ registry=test_registry)
24
+ for i in range(num_engines):
25
+ g.labels(model="foo", engine_index=str(i)).set(98.5)
26
+
27
+ metrics = get_metrics_snapshot()
28
+ assert len(metrics) == num_engines
29
+ engine_labels = [str(i) for i in range(num_engines)]
30
+ for m in metrics:
31
+ assert isinstance(m, Gauge)
32
+ assert m.name == "vllm:test_gauge"
33
+ assert m.value == 98.5
34
+ assert m.labels["model"] == "foo"
35
+ assert m.labels["engine_index"] in engine_labels
36
+ engine_labels.remove(m.labels["engine_index"])
37
+
38
+
39
+@pytest.mark.parametrize("num_engines", [1, 4])
40
+def test_counter_metric(test_registry, num_engines):
41
+ c = prometheus_client.Counter("vllm:test_counter",
42
+ "Test counter metric",
43
+ labelnames=["model", "engine_index"],
44
+ registry=test_registry)
45
+ for i in range(num_engines):
46
+ c.labels(model="bar", engine_index=str(i)).inc(19)
47
+
48
+ metrics = get_metrics_snapshot()
49
+ assert len(metrics) == num_engines
50
+ engine_labels = [str(i) for i in range(num_engines)]
51
+ for m in metrics:
52
+ assert isinstance(m, Counter)
53
+ assert m.name == "vllm:test_counter"
54
+ assert m.value == 19
55
+ assert m.labels["model"] == "bar"
56
+ assert m.labels["engine_index"] in engine_labels
57
+ engine_labels.remove(m.labels["engine_index"])
58
+
59
+
60
+@pytest.mark.parametrize("num_engines", [1, 4])
61
+def test_histogram_metric(test_registry, num_engines):
62
+ h = prometheus_client.Histogram("vllm:test_histogram",
63
+ "Test histogram metric",
64
+ labelnames=["model", "engine_index"],
65
+ buckets=[10, 20, 30, 40, 50],
66
+ registry=test_registry)
67
+ for i in range(num_engines):
68
+ hist = h.labels(model="blaa", engine_index=str(i))
69
+ hist.observe(42)
70
+ hist.observe(21)
71
+ hist.observe(7)
72
+
73
+ metrics = get_metrics_snapshot()
74
+ assert len(metrics) == num_engines
75
+ engine_labels = [str(i) for i in range(num_engines)]
76
+ for m in metrics:
77
+ assert isinstance(m, Histogram)
78
+ assert m.name == "vllm:test_histogram"
79
+ assert m.count == 3
80
+ assert m.sum == 70
81
+ assert m.buckets["10.0"] == 1
82
+ assert m.buckets["20.0"] == 1
83
+ assert m.buckets["30.0"] == 2
84
+ assert m.buckets["40.0"] == 2
85
+ assert m.buckets["50.0"] == 3
86
+ assert m.labels["model"] == "blaa"
87
+ assert m.labels["engine_index"] in engine_labels
88
+ engine_labels.remove(m.labels["engine_index"])
89
+
90
+
91
+@pytest.mark.parametrize("num_engines", [1, 4])
92
+def test_vector_metric(test_registry, num_engines):
93
+ c = prometheus_client.Counter(
94
+ "vllm:spec_decode_num_accepted_tokens_per_pos",
95
+ "Vector-like counter metric",
96
+ labelnames=["position", "model", "engine_index"],
97
+ registry=test_registry)
98
+ for i in range(num_engines):
99
+ c.labels(position="0", model="llama", engine_index=str(i)).inc(10)
100
+ c.labels(position="1", model="llama", engine_index=str(i)).inc(5)
101
+ c.labels(position="2", model="llama", engine_index=str(i)).inc(1)
102
+
103
+ metrics = get_metrics_snapshot()
104
+ assert len(metrics) == num_engines
105
+ engine_labels = [str(i) for i in range(num_engines)]
106
+ for m in metrics:
107
+ assert isinstance(m, Vector)
108
+ assert m.name == "vllm:spec_decode_num_accepted_tokens_per_pos"
109
+ assert m.values == [10, 5, 1]
110
+ assert m.labels["model"] == "llama"
111
+ assert m.labels["engine_index"] in engine_labels
112
+ engine_labels.remove(m.labels["engine_index"])
vllm/entrypoints/llm.py
+19
-1
@@ -4,7 +4,8 @@ import itertools
4
import warnings
5
from collections.abc import Sequence
6
from contextlib import contextmanager
7
-from typing import Any, Callable, ClassVar, Optional, Union, cast, overload
7
+from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Optional, Union,
8
+ cast, overload)
9
10
import cloudpickle
11
import torch.nn as nn
@@ -47,6 +48,9 @@ from vllm.usage.usage_lib import UsageContext
48
from vllm.utils import (Counter, Device, deprecate_args, deprecate_kwargs,
49
is_list_of)
50
51
+if TYPE_CHECKING:
52
+ from vllm.v1.metrics.reader import Metric
53
+
54
logger = init_logger(__name__)
55
56
_R = TypeVar("_R", default=Any)
@@ -1294,6 +1298,20 @@ class LLM:
1298
"""
1299
self.llm_engine.wake_up(tags)
1300
1301
+ def get_metrics(self) -> list["Metric"]:
1302
+ """Return a snapshot of aggregated metrics from Prometheus.
1303
+
1304
+ Returns:
1305
+ A ``MetricSnapshot`` instance capturing the current state
1306
+ of all aggregated metrics from Prometheus.
1307
+
1308
+ Note:
1309
+ This method is only available with the V1 LLM engine.
1310
+ """
1311
+ from vllm.v1.engine.llm_engine import LLMEngine as V1LLMEngine
1312
+ assert isinstance(self.llm_engine, V1LLMEngine)
1313
+ return self.llm_engine.get_metrics()
1314
+
1315
# LEGACY
1316
def _convert_v1_inputs(
1317
self,
vllm/v1/engine/llm_engine.py
+25
-4
@@ -27,7 +27,10 @@ from vllm.v1.engine.output_processor import OutputProcessor
27
from vllm.v1.engine.parallel_sampling import ParentRequest
28
from vllm.v1.engine.processor import Processor
29
from vllm.v1.executor.abstract import Executor
30
-from vllm.v1.metrics.loggers import StatLoggerFactory
30
+from vllm.v1.metrics.loggers import (PrometheusStatLogger, StatLoggerBase,
31
+ StatLoggerFactory)
32
+from vllm.v1.metrics.reader import Metric, get_metrics_snapshot
33
+from vllm.v1.metrics.stats import IterationStats
34
35
logger = init_logger(__name__)
36
@@ -64,6 +67,11 @@ class LLMEngine:
67
self.model_config = vllm_config.model_config
68
self.cache_config = vllm_config.cache_config
69
70
+ self.log_stats = log_stats
71
+ self.stat_logger: Optional[StatLoggerBase] = None
72
+ if self.log_stats:
73
+ self.stat_logger = PrometheusStatLogger(vllm_config)
74
+
75
# important: init dp group before init the engine_core
76
# In the decoupled engine case this is handled in EngineCoreProc.
77
parallel_config = vllm_config.parallel_config
@@ -86,7 +94,7 @@ class LLMEngine:
94
95
# OutputProcessor (convert EngineCoreOutputs --> RequestOutput).
96
self.output_processor = OutputProcessor(self.tokenizer,
89
- log_stats=False)
97
+ log_stats=self.log_stats)
98
99
# EngineCore (gets EngineCoreRequests and gives EngineCoreOutputs)
100
self.engine_core = EngineCoreClient.make_client(
@@ -94,7 +102,7 @@ class LLMEngine:
102
asyncio_mode=False,
103
vllm_config=vllm_config,
104
executor_class=executor_class,
97
- log_stats=False, # FIXME: implement
105
+ log_stats=self.log_stats,
106
)
107
108
if not multiprocess_mode:
@@ -223,12 +231,21 @@ class LLMEngine:
231
outputs = self.engine_core.get_output()
232
233
# 2) Process EngineCoreOutputs.
234
+ iteration_stats = IterationStats() if self.log_stats else None
235
processed_outputs = self.output_processor.process_outputs(
227
- outputs.outputs)
236
+ outputs.outputs,
237
+ engine_core_timestamp=outputs.timestamp,
238
+ iteration_stats=iteration_stats)
239
240
# 3) Abort any reqs that finished due to stop strings.
241
self.engine_core.abort_requests(processed_outputs.reqs_to_abort)
242
243
+ # 4) Record stats
244
+ if self.stat_logger is not None:
245
+ assert outputs.scheduler_stats is not None
246
+ self.stat_logger.record(scheduler_stats=outputs.scheduler_stats,
247
+ iteration_stats=iteration_stats)
248
+
249
return processed_outputs.request_outputs
250
251
def get_vllm_config(self):
@@ -260,6 +277,10 @@ class LLMEngine:
277
def is_sleeping(self) -> bool:
278
return self.engine_core.is_sleeping()
279
280
+ def get_metrics(self) -> list[Metric]:
281
+ assert self.log_stats, "Stat logging disabled"
282
+ return get_metrics_snapshot()
283
+
284
def get_tokenizer_group(self) -> TokenizerGroup:
285
if self.tokenizer is None:
286
raise ValueError("Unable to get tokenizer because "
vllm/v1/metrics/loggers.py
+4
-4
@@ -200,24 +200,24 @@ class PrometheusStatLogger(StatLoggerBase):
200
# Counters
201
#
202
self.counter_num_preempted_reqs = self._counter_cls(
203
- name="vllm:num_preemptions_total",
203
+ name="vllm:num_preemptions",
204
documentation="Cumulative number of preemption from the engine.",
205
labelnames=labelnames).labels(*labelvalues)
206
207
self.counter_prompt_tokens = self._counter_cls(
208
- name="vllm:prompt_tokens_total",
208
+ name="vllm:prompt_tokens",
209
documentation="Number of prefill tokens processed.",
210
labelnames=labelnames).labels(*labelvalues)
211
212
self.counter_generation_tokens = self._counter_cls(
213
- name="vllm:generation_tokens_total",
213
+ name="vllm:generation_tokens",
214
documentation="Number of generation tokens processed.",
215
labelnames=labelnames).labels(*labelvalues)
216
217
self.counter_request_success: dict[FinishReason,
218
prometheus_client.Counter] = {}
219
counter_request_success_base = self._counter_cls(
220
- name="vllm:request_success_total",
220
+ name="vllm:request_success",
221
documentation="Count of successfully processed requests.",
222
labelnames=labelnames + ["finished_reason"])
223
for reason in FinishReason:
vllm/v1/metrics/reader.py
new
+245
@@ -0,0 +1,245 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+from dataclasses import dataclass
4
+from typing import Optional
5
+
6
+from prometheus_client import REGISTRY
7
+from prometheus_client import Metric as PromMetric
8
+from prometheus_client.samples import Sample
9
+
10
+
11
+@dataclass
12
+class Metric:
13
+ """A base class for prometheus metrics.
14
+
15
+ Each metric may be associated with key=value labels, and
16
+ in some cases a single vLLM instance may have multiple
17
+ metrics with the same name but different sets of labels.
18
+ """
19
+ name: str
20
+ labels: dict[str, str]
21
+
22
+
23
+@dataclass
24
+class Counter(Metric):
25
+ """A monotonically increasing integer counter."""
26
+ value: int
27
+
28
+
29
+@dataclass
30
+class Vector(Metric):
31
+ """An ordered array of integer counters.
32
+
33
+ This type - which doesn't exist in Prometheus - models one very
34
+ specific metric, vllm:spec_decode_num_accepted_tokens_per_pos.
35
+ """
36
+ values: list[int]
37
+
38
+
39
+@dataclass
40
+class Gauge(Metric):
41
+ """A numerical value that can go up or down."""
42
+ value: float
43
+
44
+
45
+@dataclass
46
+class Histogram(Metric):
47
+ """Observations recorded in configurable buckets.
48
+
49
+ Buckets are represented by a dictionary. The key is
50
+ the upper limit of the bucket, and the value is the
51
+ observed count in that bucket. A '+Inf' key always
52
+ exists.
53
+
54
+ The count property is the total count across all
55
+ buckets, identical to the count of the '+Inf' bucket.
56
+
57
+ The sum property is the total sum of all observed
58
+ values.
59
+ """
60
+ count: int
61
+ sum: float
62
+ buckets: dict[str, int]
63
+
64
+
65
+def get_metrics_snapshot() -> list[Metric]:
66
+ """An API for accessing in-memory Prometheus metrics.
67
+
68
+ Example:
69
+ >>> for metric in llm.get_metrics():
70
+ ... if isinstance(metric, Counter):
71
+ ... print(f"{metric} = {metric.value}")
72
+ ... elif isinstance(metric, Gauge):
73
+ ... print(f"{metric} = {metric.value}")
74
+ ... elif isinstance(metric, Histogram):
75
+ ... print(f"{metric}")
76
+ ... print(f" sum = {metric.sum}")
77
+ ... print(f" count = {metric.count}")
78
+ ... for bucket_le, value in metrics.buckets.items():
79
+ ... print(f" {bucket_le} = {value}")
80
+ """
81
+ collected: list[Metric] = []
82
+ for metric in REGISTRY.collect():
83
+ if not metric.name.startswith("vllm:"):
84
+ continue
85
+ if metric.type == "gauge":
86
+ samples = _get_samples(metric)
87
+ for s in samples:
88
+ collected.append(
89
+ Gauge(name=metric.name, labels=s.labels, value=s.value))
90
+ elif metric.type == "counter":
91
+ samples = _get_samples(metric, "_total")
92
+ if metric.name == "vllm:spec_decode_num_accepted_tokens_per_pos":
93
+ #
94
+ # Ugly vllm:num_accepted_tokens_per_pos special case.
95
+ #
96
+ # This metric is a vector of counters - for each spec
97
+ # decoding token position, we observe the number of
98
+ # accepted tokens using a Counter labeled with 'position'.
99
+ # We convert these into a vector of integer values.
100
+ #
101
+ for labels, values in _digest_num_accepted_by_pos_samples(
102
+ samples):
103
+ collected.append(
104
+ Vector(name=metric.name, labels=labels, values=values))
105
+ else:
106
+ for s in samples:
107
+ collected.append(
108
+ Counter(name=metric.name,
109
+ labels=s.labels,
110
+ value=int(s.value)))
111
+
112
+ elif metric.type == "histogram":
113
+ #
114
+ # A histogram has a number of '_bucket' samples where
115
+ # the 'le' label represents the upper limit of the bucket.
116
+ # We convert these bucketized values into a dict of values
117
+ # indexed by the value of the 'le' label. The 'le=+Inf'
118
+ # label is a special case, catching all values observed.
119
+ #
120
+ bucket_samples = _get_samples(metric, "_bucket")
121
+ count_samples = _get_samples(metric, "_count")
122
+ sum_samples = _get_samples(metric, "_sum")
123
+ for labels, buckets, count_value, sum_value in _digest_histogram(
124
+ bucket_samples, count_samples, sum_samples):
125
+ collected.append(
126
+ Histogram(name=metric.name,
127
+ labels=labels,
128
+ buckets=buckets,
129
+ count=count_value,
130
+ sum=sum_value))
131
+ else:
132
+ raise AssertionError(f"Unknown metric type {metric.type}")
133
+
134
+ return collected
135
+
136
+
137
+def _get_samples(metric: PromMetric,
138
+ suffix: Optional[str] = None) -> list[Sample]:
139
+ name = (metric.name + suffix) if suffix is not None else metric.name
140
+ return [s for s in metric.samples if s.name == name]
141
+
142
+
143
+def _strip_label(labels: dict[str, str], key_to_remove: str) -> dict[str, str]:
144
+ labels_copy = labels.copy()
145
+ labels_copy.pop(key_to_remove)
146
+ return labels_copy
147
+
148
+
149
+def _digest_histogram(
150
+ bucket_samples: list[Sample], count_samples: list[Sample],
151
+ sum_samples: list[Sample]
152
+) -> list[tuple[dict[str, str], dict[str, int], int, float]]:
153
+ #
154
+ # In the case of DP, we have an indigestable
155
+ # per-bucket-per-engine count as a list of labelled
156
+ # samples, along with total and sum samples
157
+ #
158
+ # bucket_samples (in):
159
+ # labels = {bucket: 100, idx: 0}, value = 2
160
+ # labels = {bucket: 200, idx: 0}, value = 4
161
+ # labels = {bucket: Inf, idx: 0}, value = 10
162
+ # labels = {bucket: 100, idx: 1}, value = 1
163
+ # labels = {bucket: 200, idx: 2}, value = 5
164
+ # labels = {bucket: Inf, idx: 3}, value = 7
165
+ # count_samples (in):
166
+ # labels = {idx: 0}, value = 10
167
+ # labels = {idx: 1}, value = 7
168
+ # sum_samples (in):
169
+ # labels = {idx: 0}, value = 2000
170
+ # labels = {idx: 1}, value = 1200
171
+ #
172
+ # output: [
173
+ # {idx: 0}, {"100": 2, "200": 4, "Inf": 10}, 10, 2000
174
+ # {idx: 1}, {"100": 1, "200": 5, "Inf": 7}, 7, 1200
175
+ # ]
176
+ buckets_by_labels: dict[frozenset[tuple[str, str]], dict[str, int]] = {}
177
+ for s in bucket_samples:
178
+ bucket = s.labels["le"]
179
+ labels_key = frozenset(_strip_label(s.labels, "le").items())
180
+ if labels_key not in buckets_by_labels:
181
+ buckets_by_labels[labels_key] = {}
182
+ buckets_by_labels[labels_key][bucket] = int(s.value)
183
+
184
+ counts_by_labels: dict[frozenset[tuple[str, str]], int] = {}
185
+ for s in count_samples:
186
+ labels_key = frozenset(s.labels.items())
187
+ counts_by_labels[labels_key] = int(s.value)
188
+
189
+ sums_by_labels: dict[frozenset[tuple[str, str]], float] = {}
190
+ for s in sum_samples:
191
+ labels_key = frozenset(s.labels.items())
192
+ sums_by_labels[labels_key] = s.value
193
+
194
+ assert set(buckets_by_labels.keys()) == set(
195
+ counts_by_labels.keys()) == set(sums_by_labels.keys())
196
+
197
+ output = []
198
+ label_keys = list(buckets_by_labels.keys())
199
+ for k in label_keys:
200
+ labels = dict(k)
201
+ output.append((labels, buckets_by_labels[k], counts_by_labels[k],
202
+ sums_by_labels[k]))
203
+ return output
204
+
205
+
206
+def _digest_num_accepted_by_pos_samples(
207
+ samples: list[Sample]) -> list[tuple[dict[str, str], list[int]]]:
208
+ #
209
+ # In the case of DP, we have an indigestable
210
+ # per-position-per-engine count as a list of
211
+ # labelled samples
212
+ #
213
+ # samples (in):
214
+ # labels = {pos: 0, idx: 0}, value = 10
215
+ # labels = {pos: 1, idx: 0}, value = 7
216
+ # labels = {pos: 2, idx: 0}, value = 2
217
+ # labels = {pos: 0, idx: 1}, value = 5
218
+ # labels = {pos: 1, idx: 1}, value = 3
219
+ # labels = {pos: 2, idx: 1}, value = 1
220
+ #
221
+ # output: [
222
+ # {idx: 0}, [10, 7, 2]
223
+ # {idx: 1}, [5, 3, 1]
224
+ # ]
225
+ #
226
+ max_pos = 0
227
+ values_by_labels: dict[frozenset[tuple[str, str]], dict[int, int]] = {}
228
+
229
+ for s in samples:
230
+ position = int(s.labels["position"])
231
+ max_pos = max(max_pos, position)
232
+
233
+ labels_key = frozenset(_strip_label(s.labels, "position").items())
234
+ if labels_key not in values_by_labels:
235
+ values_by_labels[labels_key] = {}
236
+ values_by_labels[labels_key][position] = int(s.value)
237
+
238
+ output = []
239
+ for labels_key, values_by_position in values_by_labels.items():
240
+ labels = dict(labels_key)
241
+ values = [0] * (max_pos + 1)
242
+ for pos, val in values_by_position.items():
243
+ values[pos] = val
244
+ output.append((labels, values))
245
+ return output
vllm/v1/spec_decode/metrics.py
+3
-3
@@ -134,17 +134,17 @@ class SpecDecodingProm:
134
135
self.counter_spec_decode_num_drafts = \
136
self._counter_cls(
137
- name="vllm:spec_decode_num_drafts_total",
137
+ name="vllm:spec_decode_num_drafts",
138
documentation="Number of spec decoding drafts.",
139
labelnames=labelnames).labels(*labelvalues)
140
self.counter_spec_decode_num_draft_tokens = \
141
self._counter_cls(
142
- name="vllm:spec_decode_num_draft_tokens_total",
142
+ name="vllm:spec_decode_num_draft_tokens",
143
documentation="Number of draft tokens.",
144
labelnames=labelnames,).labels(*labelvalues)
145
self.counter_spec_decode_num_accepted_tokens = \
146
self._counter_cls(
147
- name="vllm:spec_decode_num_accepted_tokens_total",
147
+ name="vllm:spec_decode_num_accepted_tokens",
148
documentation="Number of accepted tokens.",
149
labelnames=labelnames).labels(*labelvalues)
150