A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1][Metrics] Add TTFT and TPOT histograms (#12530)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
Jan 29, 2025 at 04:11 UTC
46fb056749b7d9f5e4ea7a060207ed2eb3ad75e0
4 files changed
+45
-1
tests/entrypoints/openai/test_metrics.py
+6
@@ -208,6 +208,12 @@ EXPECTED_METRICS_V1 = [
208
"vllm:request_generation_tokens_sum",
209
"vllm:request_generation_tokens_bucket",
210
"vllm:request_generation_tokens_count",
211
+ "vllm:time_to_first_token_seconds_sum",
212
+ "vllm:time_to_first_token_seconds_bucket",
213
+ "vllm:time_to_first_token_seconds_count",
214
+ "vllm:time_per_output_token_seconds_sum",
215
+ "vllm:time_per_output_token_seconds_bucket",
216
+ "vllm:time_per_output_token_seconds_count",
217
]
218
219
vllm/v1/engine/output_processor.py
+3
-1
@@ -27,6 +27,7 @@ class RequestState:
27
prompt: Optional[str],
28
prompt_token_ids: List[int],
29
detokenizer: IncrementalDetokenizer,
30
+ arrival_time: float,
31
queue: Optional[asyncio.Queue[RequestOutput]],
32
):
33
self.request_id = request_id
@@ -37,7 +38,7 @@ class RequestState:
38
self.is_prefilling = True
39
self.queue = queue
40
40
- self.stats = RequestStateStats()
41
+ self.stats = RequestStateStats(last_token_time=arrival_time)
42
43
@classmethod
44
def from_new_request(
@@ -54,6 +55,7 @@ class RequestState:
55
tokenizer=tokenizer,
56
request=request,
57
),
58
+ arrival_time=request.arrival_time,
59
queue=queue,
60
)
61
vllm/v1/metrics/loggers.py
+25
@@ -121,6 +121,26 @@ class PrometheusStatLogger(StatLoggerBase):
121
buckets=build_1_2_5_buckets(max_model_len),
122
labelnames=labelnames).labels(*labelvalues)
123
124
+ self.histogram_time_to_first_token = \
125
+ prometheus_client.Histogram(
126
+ name="vllm:time_to_first_token_seconds",
127
+ documentation="Histogram of time to first token in seconds.",
128
+ buckets=[
129
+ 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
130
+ 0.75, 1.0, 2.5, 5.0, 7.5, 10.0
131
+ ],
132
+ labelnames=labelnames).labels(*labelvalues)
133
+
134
+ self.histogram_time_per_output_token = \
135
+ prometheus_client.Histogram(
136
+ name="vllm:time_per_output_token_seconds",
137
+ documentation="Histogram of time per output token in seconds.",
138
+ buckets=[
139
+ 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5,
140
+ 0.75, 1.0, 2.5
141
+ ],
142
+ labelnames=labelnames).labels(*labelvalues)
143
+
144
def log(self, scheduler_stats: SchedulerStats,
145
iteration_stats: IterationStats):
146
"""Log to prometheus."""
@@ -137,6 +157,11 @@ class PrometheusStatLogger(StatLoggerBase):
157
self.histogram_num_generation_tokens_request.observe(
158
finished_request.num_generation_tokens)
159
160
+ for ttft in iteration_stats.time_to_first_tokens_iter:
161
+ self.histogram_time_to_first_token.observe(ttft)
162
+ for tpot in iteration_stats.time_per_output_tokens_iter:
163
+ self.histogram_time_per_output_token.observe(tpot)
164
+
165
@staticmethod
166
def _unregister_vllm_metrics():
167
# Unregister any existing vLLM collectors (for CI/CD
vllm/v1/metrics/stats.py
+11
@@ -1,3 +1,4 @@
1
+import time
2
from dataclasses import dataclass
3
from typing import TYPE_CHECKING, List
4
@@ -22,6 +23,7 @@ class RequestStateStats:
23
"""Stats that need to be tracked across delta updates."""
24
25
num_generation_tokens: int = 0
26
+ last_token_time: float = 0.0
27
28
29
@dataclass
@@ -40,6 +42,8 @@ class IterationStats:
42
self.num_generation_tokens = 0
43
self.num_prompt_tokens = 0
44
self.finished_requests: List[FinishedRequestStats] = []
45
+ self.time_to_first_tokens_iter: List[float] = []
46
+ self.time_per_output_tokens_iter: List[float] = []
47
48
def update_from_output(self, output: "EngineCoreOutput",
49
is_prefilling: bool, prompt_len: int,
@@ -48,6 +52,8 @@ class IterationStats:
52
return
53
54
num_new_generation_tokens = len(output.new_token_ids)
55
+ now = time.time()
56
+ last_token_latency = now - request_state_stats.last_token_time
57
58
self.num_generation_tokens += num_new_generation_tokens
59
if is_prefilling:
@@ -58,7 +64,12 @@ class IterationStats:
64
assert (num_new_generation_tokens > 0)
65
self.num_prompt_tokens += prompt_len
66
67
+ self.time_to_first_tokens_iter.append(last_token_latency)
68
+ else:
69
+ self.time_per_output_tokens_iter.append(last_token_latency)
70
+
71
request_state_stats.num_generation_tokens += num_new_generation_tokens
72
+ request_state_stats.last_token_time = now
73
74
def update_from_finished_request(self, request_output: "RequestOutput",
75
request_state_stats: RequestStateStats):