A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Frontend] Add max_tokens prometheus metric (#9881)
Signed-off-by: Tomer Asida <tomera@ai21.com>
tomeras91 committed
Nov 5, 2024 at 00:53 UTC
ac04a97a9fbc122bb14ff4eb590314d453cdf57c
5 files changed
+23
-2
tests/entrypoints/openai/test_metrics.py
+9
-2
@@ -70,10 +70,14 @@ EXPECTED_VALUES = {
70
[("_sum", _NUM_REQUESTS * _NUM_GENERATION_TOKENS_PER_REQUEST),
71
("_count", _NUM_REQUESTS)],
72
"vllm:request_params_n": [("_count", _NUM_REQUESTS)],
73
+ "vllm:request_params_max_tokens":
74
+ [("_sum", _NUM_REQUESTS * _NUM_GENERATION_TOKENS_PER_REQUEST),
75
+ ("_count", _NUM_REQUESTS)],
76
"vllm:prompt_tokens": [("_total",
77
_NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)],
75
- "vllm:generation_tokens":
76
- [("_total", _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)],
78
+ "vllm:generation_tokens": [
79
+ ("_total", _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)
80
+ ],
81
"vllm:request_success": [("_total", _NUM_REQUESTS)],
82
}
83
@@ -149,6 +153,9 @@ EXPECTED_METRICS = [
153
"vllm:request_params_n_sum",
154
"vllm:request_params_n_bucket",
155
"vllm:request_params_n_count",
156
+ "vllm:request_params_max_tokens_sum",
157
+ "vllm:request_params_max_tokens_bucket",
158
+ "vllm:request_params_max_tokens_count",
159
"vllm:num_preemptions_total",
160
"vllm:prompt_tokens_total",
161
"vllm:generation_tokens_total",
tests/metrics/test_metrics.py
+1
@@ -365,6 +365,7 @@ def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
365
"vllm:request_prompt_tokens",
366
"vllm:request_generation_tokens",
367
"vllm:request_params_n",
368
+ "vllm:request_params_max_tokens",
369
]
370
for metric_name in request_histogram_metrics:
371
metric_value = REGISTRY.get_sample_value(f"{metric_name}_count",
vllm/engine/llm_engine.py
+4
@@ -1685,6 +1685,7 @@ class LLMEngine:
1685
num_prompt_tokens_requests: List[int] = []
1686
num_generation_tokens_requests: List[int] = []
1687
n_requests: List[int] = []
1688
+ max_tokens_requests: List[int] = []
1689
finished_reason_requests: List[str] = []
1690
1691
# Lora requests
@@ -1792,6 +1793,8 @@ class LLMEngine:
1793
])
1794
if seq_group.sampling_params is not None:
1795
n_requests.append(seq_group.sampling_params.n)
1796
+ max_tokens_requests.append(
1797
+ seq_group.sampling_params.max_tokens)
1798
finished_reason_requests.extend([
1799
SequenceStatus.get_finished_reason(seq.status)
1800
for seq in seq_group.get_finished_seqs()
@@ -1847,6 +1850,7 @@ class LLMEngine:
1850
num_prompt_tokens_requests=num_prompt_tokens_requests,
1851
num_generation_tokens_requests=num_generation_tokens_requests,
1852
n_requests=n_requests,
1853
+ max_tokens_requests=max_tokens_requests,
1854
finished_reason_requests=finished_reason_requests,
1855
max_lora=str(max_lora_stat),
1856
waiting_lora_adapters=list(waiting_lora_adapters.keys()),
vllm/engine/metrics.py
+8
@@ -179,6 +179,12 @@ class Metrics:
179
labelnames=labelnames,
180
buckets=[1, 2, 5, 10, 20],
181
)
182
+ self.histogram_max_tokens_request = self._histogram_cls(
183
+ name="vllm:request_params_max_tokens",
184
+ documentation="Histogram of the max_tokens request parameter.",
185
+ labelnames=labelnames,
186
+ buckets=build_1_2_5_buckets(max_model_len),
187
+ )
188
self.counter_request_success = self._counter_cls(
189
name="vllm:request_success_total",
190
documentation="Count of successfully processed requests.",
@@ -547,6 +553,8 @@ class PrometheusStatLogger(StatLoggerBase):
553
self.metrics.histogram_num_generation_tokens_request,
554
stats.num_generation_tokens_requests)
555
self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
556
+ self._log_histogram(self.metrics.histogram_max_tokens_request,
557
+ stats.max_tokens_requests)
558
559
def _log_prometheus_interval(self, prompt_throughput: float,
560
generation_throughput: float) -> None:
vllm/engine/metrics_types.py
+1
@@ -53,6 +53,7 @@ class Stats:
53
num_prompt_tokens_requests: List[int]
54
num_generation_tokens_requests: List[int]
55
n_requests: List[int]
56
+ max_tokens_requests: List[int]
57
finished_reason_requests: List[str]
58
waiting_lora_adapters: List[str]
59
running_lora_adapters: List[str]