@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c386c43ca3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][Metrics] Add per-request prompt/generation_tokens histograms (#12516)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>

Mark McLoughlin committed Jan 28, 2025 at 22:07 UTC c386c43ca3a7156a953e0ca4d8f2c2f36ccf1423
5 files changed +102 -14
tests/entrypoints/openai/test_metrics.py
+6
@@ -202,6 +202,12 @@ EXPECTED_METRICS_V1 = [
202 "vllm:num_requests_waiting",
203 "vllm:prompt_tokens_total",
204 "vllm:generation_tokens_total",
205 + "vllm:request_prompt_tokens_sum",
206 + "vllm:request_prompt_tokens_bucket",
207 + "vllm:request_prompt_tokens_count",
208 + "vllm:request_generation_tokens_sum",
209 + "vllm:request_generation_tokens_bucket",
210 + "vllm:request_generation_tokens_count",
211 ]
212
213
vllm/v1/engine/async_llm.py
+1 -2
@@ -53,8 +53,7 @@ class AsyncLLM(EngineClient):
53 self.log_stats = log_stats
54 self.stat_loggers: List[StatLoggerBase] = [
55 LoggingStatLogger(),
56 - PrometheusStatLogger(labels=dict(
57 - model_name=self.model_config.served_model_name)),
56 + PrometheusStatLogger(vllm_config.model_config),
57 ]
58
59 # Tokenizer (+ ensure liveness if running in another process).
vllm/v1/engine/output_processor.py
+9 -2
@@ -8,7 +8,7 @@ from vllm.transformers_utils.tokenizer_group import BaseTokenizerGroup
8 from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest
9 from vllm.v1.engine.detokenizer import (DetokenizerOutput,
10 IncrementalDetokenizer)
11 -from vllm.v1.metrics.stats import IterationStats
11 +from vllm.v1.metrics.stats import IterationStats, RequestStateStats
12
13
14 @dataclass
@@ -37,6 +37,8 @@ class RequestState:
37 self.is_prefilling = True
38 self.queue = queue
39
40 + self.stats = RequestStateStats()
41 +
42 @classmethod
43 def from_new_request(
44 cls,
@@ -146,7 +148,8 @@ class OutputProcessor:
148 # 1) Compute stats for this iteration.
149 iteration_stats.update_from_output(engine_core_output,
150 req_state.is_prefilling,
149 - req_state.prompt_len)
151 + req_state.prompt_len,
152 + req_state.stats)
153 req_state.is_prefilling = False
154
155 # 2) Detokenize the token ids into text.
@@ -171,6 +174,10 @@ class OutputProcessor:
174 # detected stop string, abort needed in EngineCore.
175 reqs_to_abort.append(req_id)
176
177 + # Track per-request stats
178 + iteration_stats.update_from_finished_request(
179 + request_output, req_state.stats)
180 +
181 return OutputProcessorOutput(
182 request_outputs=request_outputs,
183 reqs_to_abort=reqs_to_abort,
vllm/v1/metrics/loggers.py
+54 -6
@@ -1,10 +1,11 @@
1 import time
2 from abc import ABC, abstractmethod
3 -from typing import Dict, List
3 +from typing import List
4
5 import numpy as np
6 import prometheus_client
7
8 +from vllm.config import ModelConfig
9 from vllm.logger import init_logger
10 from vllm.v1.metrics.stats import IterationStats, SchedulerStats
11
@@ -78,13 +79,13 @@ class LoggingStatLogger(StatLoggerBase):
79
80 class PrometheusStatLogger(StatLoggerBase):
81
81 - def __init__(self, labels: Dict[str, str]):
82 - self.labels = labels
82 + def __init__(self, model_config: ModelConfig):
83 + self._unregister_vllm_metrics()
84
84 - labelnames = self.labels.keys()
85 - labelvalues = self.labels.values()
85 + labelnames = ["model_name"]
86 + labelvalues = [model_config.served_model_name]
87
87 - self._unregister_vllm_metrics()
88 + max_model_len = model_config.max_model_len
89
90 self.gauge_scheduler_running = prometheus_client.Gauge(
91 name="vllm:num_requests_running",
@@ -106,6 +107,20 @@ class PrometheusStatLogger(StatLoggerBase):
107 documentation="Number of generation tokens processed.",
108 labelnames=labelnames).labels(*labelvalues)
109
110 + self.histogram_num_prompt_tokens_request = \
111 + prometheus_client.Histogram(
112 + name="vllm:request_prompt_tokens",
113 + documentation="Number of prefill tokens processed.",
114 + buckets=build_1_2_5_buckets(max_model_len),
115 + labelnames=labelnames).labels(*labelvalues)
116 +
117 + self.histogram_num_generation_tokens_request = \
118 + prometheus_client.Histogram(
119 + name="vllm:request_generation_tokens",
120 + documentation="Number of generation tokens processed.",
121 + buckets=build_1_2_5_buckets(max_model_len),
122 + labelnames=labelnames).labels(*labelvalues)
123 +
124 def log(self, scheduler_stats: SchedulerStats,
125 iteration_stats: IterationStats):
126 """Log to prometheus."""
@@ -116,9 +131,42 @@ class PrometheusStatLogger(StatLoggerBase):
131 self.counter_generation_tokens.inc(
132 iteration_stats.num_generation_tokens)
133
134 + for finished_request in iteration_stats.finished_requests:
135 + self.histogram_num_prompt_tokens_request.observe(
136 + finished_request.num_prompt_tokens)
137 + self.histogram_num_generation_tokens_request.observe(
138 + finished_request.num_generation_tokens)
139 +
140 @staticmethod
141 def _unregister_vllm_metrics():
142 # Unregister any existing vLLM collectors (for CI/CD
143 for collector in list(prometheus_client.REGISTRY._collector_to_names):
144 if hasattr(collector, "_name") and "vllm" in collector._name:
145 prometheus_client.REGISTRY.unregister(collector)
146 +
147 +
148 +def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:
149 + """
150 + Builds a list of buckets with increasing powers of 10 multiplied by
151 + mantissa values until the value exceeds the specified maximum.
152 +
153 + """
154 + exponent = 0
155 + buckets: List[int] = []
156 + while True:
157 + for m in mantissa_lst:
158 + value = m * 10**exponent
159 + if value <= max_value:
160 + buckets.append(value)
161 + else:
162 + return buckets
163 + exponent += 1
164 +
165 +
166 +def build_1_2_5_buckets(max_value: int) -> List[int]:
167 + """
168 + Example:
169 + >>> build_1_2_5_buckets(100)
170 + [1, 2, 5, 10, 20, 50, 100]
171 + """
172 + return build_buckets([1, 2, 5], max_value)
vllm/v1/metrics/stats.py
+32 -4
@@ -1,7 +1,8 @@
1 from dataclasses import dataclass
2 -from typing import TYPE_CHECKING
2 +from typing import TYPE_CHECKING, List
3
4 if TYPE_CHECKING:
5 + from vllm.outputs import RequestOutput
6 from vllm.v1.engine import EngineCoreOutput
7
8
@@ -16,6 +17,21 @@ class SchedulerStats:
17 # gpu_prefix_cache_hit_rate: float = 0.0
18
19
20 +@dataclass
21 +class RequestStateStats:
22 + """Stats that need to be tracked across delta updates."""
23 +
24 + num_generation_tokens: int = 0
25 +
26 +
27 +@dataclass
28 +class FinishedRequestStats:
29 + """Stats associated with a finished request."""
30 +
31 + num_prompt_tokens: int = 0
32 + num_generation_tokens: int = 0
33 +
34 +
35 class IterationStats:
36 """Stats associated with a single set of EngineCoreOutputs."""
37
@@ -23,17 +39,29 @@ class IterationStats:
39 self.log_stats = log_stats
40 self.num_generation_tokens = 0
41 self.num_prompt_tokens = 0
42 + self.finished_requests: List[FinishedRequestStats] = []
43
44 def update_from_output(self, output: "EngineCoreOutput",
28 - is_prefilling: bool, prompt_len: int):
45 + is_prefilling: bool, prompt_len: int,
46 + request_state_stats: RequestStateStats):
47 if not self.log_stats:
48 return
49
32 - self.num_generation_tokens += len(output.new_token_ids)
50 + num_new_generation_tokens = len(output.new_token_ids)
51 +
52 + self.num_generation_tokens += num_new_generation_tokens
53 if is_prefilling:
54 # This relies on the invariant that EngineCore does
55 # not stream outputs for partially completed prefills
56 # (scheduler.update_from_output makes EngineCoreOutput
57 # iff num_computed_tokens == num_tokens).
38 - assert (len(output.new_token_ids) > 0)
58 + assert (num_new_generation_tokens > 0)
59 self.num_prompt_tokens += prompt_len
60 +
61 + request_state_stats.num_generation_tokens += num_new_generation_tokens
62 +
63 + def update_from_finished_request(self, request_output: "RequestOutput",
64 + request_state_stats: RequestStateStats):
65 + self.finished_requests.append(
66 + FinishedRequestStats(len(request_output.prompt_token_ids),
67 + request_state_stats.num_generation_tokens))