@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 46fb056749
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][Metrics] Add TTFT and TPOT histograms (#12530)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>

Mark McLoughlin committed Jan 29, 2025 at 04:11 UTC 46fb056749b7d9f5e4ea7a060207ed2eb3ad75e0
4 files changed +45 -1
tests/entrypoints/openai/test_metrics.py
+6
@@ -208,6 +208,12 @@ EXPECTED_METRICS_V1 = [
208 "vllm:request_generation_tokens_sum",
209 "vllm:request_generation_tokens_bucket",
210 "vllm:request_generation_tokens_count",
211 + "vllm:time_to_first_token_seconds_sum",
212 + "vllm:time_to_first_token_seconds_bucket",
213 + "vllm:time_to_first_token_seconds_count",
214 + "vllm:time_per_output_token_seconds_sum",
215 + "vllm:time_per_output_token_seconds_bucket",
216 + "vllm:time_per_output_token_seconds_count",
217 ]
218
219
vllm/v1/engine/output_processor.py
+3 -1
@@ -27,6 +27,7 @@ class RequestState:
27 prompt: Optional[str],
28 prompt_token_ids: List[int],
29 detokenizer: IncrementalDetokenizer,
30 + arrival_time: float,
31 queue: Optional[asyncio.Queue[RequestOutput]],
32 ):
33 self.request_id = request_id
@@ -37,7 +38,7 @@ class RequestState:
38 self.is_prefilling = True
39 self.queue = queue
40
40 - self.stats = RequestStateStats()
41 + self.stats = RequestStateStats(last_token_time=arrival_time)
42
43 @classmethod
44 def from_new_request(
@@ -54,6 +55,7 @@ class RequestState:
55 tokenizer=tokenizer,
56 request=request,
57 ),
58 + arrival_time=request.arrival_time,
59 queue=queue,
60 )
61
vllm/v1/metrics/loggers.py
+25
@@ -121,6 +121,26 @@ class PrometheusStatLogger(StatLoggerBase):
121 buckets=build_1_2_5_buckets(max_model_len),
122 labelnames=labelnames).labels(*labelvalues)
123
124 + self.histogram_time_to_first_token = \
125 + prometheus_client.Histogram(
126 + name="vllm:time_to_first_token_seconds",
127 + documentation="Histogram of time to first token in seconds.",
128 + buckets=[
129 + 0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
130 + 0.75, 1.0, 2.5, 5.0, 7.5, 10.0
131 + ],
132 + labelnames=labelnames).labels(*labelvalues)
133 +
134 + self.histogram_time_per_output_token = \
135 + prometheus_client.Histogram(
136 + name="vllm:time_per_output_token_seconds",
137 + documentation="Histogram of time per output token in seconds.",
138 + buckets=[
139 + 0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5,
140 + 0.75, 1.0, 2.5
141 + ],
142 + labelnames=labelnames).labels(*labelvalues)
143 +
144 def log(self, scheduler_stats: SchedulerStats,
145 iteration_stats: IterationStats):
146 """Log to prometheus."""
@@ -137,6 +157,11 @@ class PrometheusStatLogger(StatLoggerBase):
157 self.histogram_num_generation_tokens_request.observe(
158 finished_request.num_generation_tokens)
159
160 + for ttft in iteration_stats.time_to_first_tokens_iter:
161 + self.histogram_time_to_first_token.observe(ttft)
162 + for tpot in iteration_stats.time_per_output_tokens_iter:
163 + self.histogram_time_per_output_token.observe(tpot)
164 +
165 @staticmethod
166 def _unregister_vllm_metrics():
167 # Unregister any existing vLLM collectors (for CI/CD
vllm/v1/metrics/stats.py
+11
@@ -1,3 +1,4 @@
1 +import time
2 from dataclasses import dataclass
3 from typing import TYPE_CHECKING, List
4
@@ -22,6 +23,7 @@ class RequestStateStats:
23 """Stats that need to be tracked across delta updates."""
24
25 num_generation_tokens: int = 0
26 + last_token_time: float = 0.0
27
28
29 @dataclass
@@ -40,6 +42,8 @@ class IterationStats:
42 self.num_generation_tokens = 0
43 self.num_prompt_tokens = 0
44 self.finished_requests: List[FinishedRequestStats] = []
45 + self.time_to_first_tokens_iter: List[float] = []
46 + self.time_per_output_tokens_iter: List[float] = []
47
48 def update_from_output(self, output: "EngineCoreOutput",
49 is_prefilling: bool, prompt_len: int,
@@ -48,6 +52,8 @@ class IterationStats:
52 return
53
54 num_new_generation_tokens = len(output.new_token_ids)
55 + now = time.time()
56 + last_token_latency = now - request_state_stats.last_token_time
57
58 self.num_generation_tokens += num_new_generation_tokens
59 if is_prefilling:
@@ -58,7 +64,12 @@ class IterationStats:
64 assert (num_new_generation_tokens > 0)
65 self.num_prompt_tokens += prompt_len
66
67 + self.time_to_first_tokens_iter.append(last_token_latency)
68 + else:
69 + self.time_per_output_tokens_iter.append(last_token_latency)
70 +
71 request_state_stats.num_generation_tokens += num_new_generation_tokens
72 + request_state_stats.last_token_time = now
73
74 def update_from_finished_request(self, request_output: "RequestOutput",
75 request_state_stats: RequestStateStats):