A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1][Metrics] Add per-request prompt/generation_tokens histograms (#12516)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
Jan 28, 2025 at 22:07 UTC
c386c43ca3a7156a953e0ca4d8f2c2f36ccf1423
5 files changed
+102
-14
tests/entrypoints/openai/test_metrics.py
+6
@@ -202,6 +202,12 @@ EXPECTED_METRICS_V1 = [
202
"vllm:num_requests_waiting",
203
"vllm:prompt_tokens_total",
204
"vllm:generation_tokens_total",
205
+ "vllm:request_prompt_tokens_sum",
206
+ "vllm:request_prompt_tokens_bucket",
207
+ "vllm:request_prompt_tokens_count",
208
+ "vllm:request_generation_tokens_sum",
209
+ "vllm:request_generation_tokens_bucket",
210
+ "vllm:request_generation_tokens_count",
211
]
212
213
vllm/v1/engine/async_llm.py
+1
-2
@@ -53,8 +53,7 @@ class AsyncLLM(EngineClient):
53
self.log_stats = log_stats
54
self.stat_loggers: List[StatLoggerBase] = [
55
LoggingStatLogger(),
56
- PrometheusStatLogger(labels=dict(
57
- model_name=self.model_config.served_model_name)),
56
+ PrometheusStatLogger(vllm_config.model_config),
57
]
58
59
# Tokenizer (+ ensure liveness if running in another process).
vllm/v1/engine/output_processor.py
+9
-2
@@ -8,7 +8,7 @@ from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
8
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
9
from vllm.v1.engine.detokenizer import (DetokenizerOutput,
10
IncrementalDetokenizer)
11
-from vllm.v1.metrics.stats import IterationStats
11
+from vllm.v1.metrics.stats import IterationStats, RequestStateStats
12
13
14
@dataclass
@@ -37,6 +37,8 @@ class RequestState:
37
self.is_prefilling = True
38
self.queue = queue
39
40
+ self.stats = RequestStateStats()
41
+
42
@classmethod
43
def from_new_request(
44
cls,
@@ -146,7 +148,8 @@ class OutputProcessor:
148
# 1) Compute stats for this iteration.
149
iteration_stats.update_from_output(engine_core_output,
150
req_state.is_prefilling,
149
- req_state.prompt_len)
151
+ req_state.prompt_len,
152
+ req_state.stats)
153
req_state.is_prefilling = False
154
155
# 2) Detokenize the token ids into text.
@@ -171,6 +174,10 @@ class OutputProcessor:
174
# detected stop string, abort needed in EngineCore.
175
reqs_to_abort.append(req_id)
176
177
+ # Track per-request stats
178
+ iteration_stats.update_from_finished_request(
179
+ request_output, req_state.stats)
180
+
181
return OutputProcessorOutput(
182
request_outputs=request_outputs,
183
reqs_to_abort=reqs_to_abort,
vllm/v1/metrics/loggers.py
+54
-6
@@ -1,10 +1,11 @@
1
import time
2
from abc import ABC, abstractmethod
3
-from typing import Dict, List
3
+from typing import List
4
5
import numpy as np
6
import prometheus_client
7
8
+from vllm.config import ModelConfig
9
from vllm.logger import init_logger
10
from vllm.v1.metrics.stats import IterationStats, SchedulerStats
11
@@ -78,13 +79,13 @@ class LoggingStatLogger(StatLoggerBase):
79
80
class PrometheusStatLogger(StatLoggerBase):
81
81
- def __init__(self, labels: Dict[str, str]):
82
- self.labels = labels
82
+ def __init__(self, model_config: ModelConfig):
83
+ self._unregister_vllm_metrics()
84
84
- labelnames = self.labels.keys()
85
- labelvalues = self.labels.values()
85
+ labelnames = ["model_name"]
86
+ labelvalues = [model_config.served_model_name]
87
87
- self._unregister_vllm_metrics()
88
+ max_model_len = model_config.max_model_len
89
90
self.gauge_scheduler_running = prometheus_client.Gauge(
91
name="vllm:num_requests_running",
@@ -106,6 +107,20 @@ class PrometheusStatLogger(StatLoggerBase):
107
documentation="Number of generation tokens processed.",
108
labelnames=labelnames).labels(*labelvalues)
109
110
+ self.histogram_num_prompt_tokens_request = \
111
+ prometheus_client.Histogram(
112
+ name="vllm:request_prompt_tokens",
113
+ documentation="Number of prefill tokens processed.",
114
+ buckets=build_1_2_5_buckets(max_model_len),
115
+ labelnames=labelnames).labels(*labelvalues)
116
+
117
+ self.histogram_num_generation_tokens_request = \
118
+ prometheus_client.Histogram(
119
+ name="vllm:request_generation_tokens",
120
+ documentation="Number of generation tokens processed.",
121
+ buckets=build_1_2_5_buckets(max_model_len),
122
+ labelnames=labelnames).labels(*labelvalues)
123
+
124
def log(self, scheduler_stats: SchedulerStats,
125
iteration_stats: IterationStats):
126
"""Log to prometheus."""
@@ -116,9 +131,42 @@ class PrometheusStatLogger(StatLoggerBase):
131
self.counter_generation_tokens.inc(
132
iteration_stats.num_generation_tokens)
133
134
+ for finished_request in iteration_stats.finished_requests:
135
+ self.histogram_num_prompt_tokens_request.observe(
136
+ finished_request.num_prompt_tokens)
137
+ self.histogram_num_generation_tokens_request.observe(
138
+ finished_request.num_generation_tokens)
139
+
140
@staticmethod
141
def _unregister_vllm_metrics():
142
# Unregister any existing vLLM collectors (for CI/CD
143
for collector in list(prometheus_client.REGISTRY._collector_to_names):
144
if hasattr(collector, "_name") and "vllm" in collector._name:
145
prometheus_client.REGISTRY.unregister(collector)
146
+
147
+
148
+def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:
149
+ """
150
+ Builds a list of buckets with increasing powers of 10 multiplied by
151
+ mantissa values until the value exceeds the specified maximum.
152
+
153
+ """
154
+ exponent = 0
155
+ buckets: List[int] = []
156
+ while True:
157
+ for m in mantissa_lst:
158
+ value = m * 10**exponent
159
+ if value <= max_value:
160
+ buckets.append(value)
161
+ else:
162
+ return buckets
163
+ exponent += 1
164
+
165
+
166
+def build_1_2_5_buckets(max_value: int) -> List[int]:
167
+ """
168
+ Example:
169
+ >>> build_1_2_5_buckets(100)
170
+ [1, 2, 5, 10, 20, 50, 100]
171
+ """
172
+ return build_buckets([1, 2, 5], max_value)
vllm/v1/metrics/stats.py
+32
-4
@@ -1,7 +1,8 @@
1
from dataclasses import dataclass
2
-from typing import TYPE_CHECKING
2
+from typing import TYPE_CHECKING, List
3
4
if TYPE_CHECKING:
5
+ from vllm.outputs import RequestOutput
6
from vllm.v1.engine import EngineCoreOutput
7
8
@@ -16,6 +17,21 @@ class SchedulerStats:
17
# gpu_prefix_cache_hit_rate: float = 0.0
18
19
20
+@dataclass
21
+class RequestStateStats:
22
+ """Stats that need to be tracked across delta updates."""
23
+
24
+ num_generation_tokens: int = 0
25
+
26
+
27
+@dataclass
28
+class FinishedRequestStats:
29
+ """Stats associated with a finished request."""
30
+
31
+ num_prompt_tokens: int = 0
32
+ num_generation_tokens: int = 0
33
+
34
+
35
class IterationStats:
36
"""Stats associated with a single set of EngineCoreOutputs."""
37
@@ -23,17 +39,29 @@ class IterationStats:
39
self.log_stats = log_stats
40
self.num_generation_tokens = 0
41
self.num_prompt_tokens = 0
42
+ self.finished_requests: List[FinishedRequestStats] = []
43
44
def update_from_output(self, output: "EngineCoreOutput",
28
- is_prefilling: bool, prompt_len: int):
45
+ is_prefilling: bool, prompt_len: int,
46
+ request_state_stats: RequestStateStats):
47
if not self.log_stats:
48
return
49
32
- self.num_generation_tokens += len(output.new_token_ids)
50
+ num_new_generation_tokens = len(output.new_token_ids)
51
+
52
+ self.num_generation_tokens += num_new_generation_tokens
53
if is_prefilling:
54
# This relies on the invariant that EngineCore does
55
# not stream outputs for partially completed prefills
56
# (scheduler.update_from_output makes EngineCoreOutput
57
# iff num_computed_tokens == num_tokens).
38
- assert (len(output.new_token_ids) > 0)
58
+ assert (num_new_generation_tokens > 0)
59
self.num_prompt_tokens += prompt_len
60
+
61
+ request_state_stats.num_generation_tokens += num_new_generation_tokens
62
+
63
+ def update_from_finished_request(self, request_output: "RequestOutput",
64
+ request_state_stats: RequestStateStats):
65
+ self.finished_requests.append(
66
+ FinishedRequestStats(len(request_output.prompt_token_ids),
67
+ request_state_stats.num_generation_tokens))