@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ac04a97a9f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Frontend] Add max_tokens prometheus metric (#9881)

Signed-off-by: Tomer Asida <tomera@ai21.com>

tomeras91 committed Nov 5, 2024 at 00:53 UTC ac04a97a9fbc122bb14ff4eb590314d453cdf57c
5 files changed +23 -2
tests/entrypoints/openai/test_metrics.py
+9 -2
@@ -70,10 +70,14 @@ EXPECTED_VALUES = {
70 [("_sum", _NUM_REQUESTS * _NUM_GENERATION_TOKENS_PER_REQUEST),
71 ("_count", _NUM_REQUESTS)],
72 "vllm:request_params_n": [("_count", _NUM_REQUESTS)],
73 + "vllm:request_params_max_tokens":
74 + [("_sum", _NUM_REQUESTS * _NUM_GENERATION_TOKENS_PER_REQUEST),
75 + ("_count", _NUM_REQUESTS)],
76 "vllm:prompt_tokens": [("_total",
77 _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)],
75 - "vllm:generation_tokens":
76 - [("_total", _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)],
78 + "vllm:generation_tokens": [
79 + ("_total", _NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)
80 + ],
81 "vllm:request_success": [("_total", _NUM_REQUESTS)],
82 }
83
@@ -149,6 +153,9 @@ EXPECTED_METRICS = [
153 "vllm:request_params_n_sum",
154 "vllm:request_params_n_bucket",
155 "vllm:request_params_n_count",
156 + "vllm:request_params_max_tokens_sum",
157 + "vllm:request_params_max_tokens_bucket",
158 + "vllm:request_params_max_tokens_count",
159 "vllm:num_preemptions_total",
160 "vllm:prompt_tokens_total",
161 "vllm:generation_tokens_total",
tests/metrics/test_metrics.py
+1
@@ -365,6 +365,7 @@ def assert_metrics(engine: LLMEngine, disable_log_stats: bool,
365 "vllm:request_prompt_tokens",
366 "vllm:request_generation_tokens",
367 "vllm:request_params_n",
368 + "vllm:request_params_max_tokens",
369 ]
370 for metric_name in request_histogram_metrics:
371 metric_value = REGISTRY.get_sample_value(f"{metric_name}_count",
vllm/engine/llm_engine.py
+4
@@ -1685,6 +1685,7 @@ class LLMEngine:
1685 num_prompt_tokens_requests: List[int] = []
1686 num_generation_tokens_requests: List[int] = []
1687 n_requests: List[int] = []
1688 + max_tokens_requests: List[int] = []
1689 finished_reason_requests: List[str] = []
1690
1691 # Lora requests
@@ -1792,6 +1793,8 @@ class LLMEngine:
1793 ])
1794 if seq_group.sampling_params is not None:
1795 n_requests.append(seq_group.sampling_params.n)
1796 + max_tokens_requests.append(
1797 + seq_group.sampling_params.max_tokens)
1798 finished_reason_requests.extend([
1799 SequenceStatus.get_finished_reason(seq.status)
1800 for seq in seq_group.get_finished_seqs()
@@ -1847,6 +1850,7 @@ class LLMEngine:
1850 num_prompt_tokens_requests=num_prompt_tokens_requests,
1851 num_generation_tokens_requests=num_generation_tokens_requests,
1852 n_requests=n_requests,
1853 + max_tokens_requests=max_tokens_requests,
1854 finished_reason_requests=finished_reason_requests,
1855 max_lora=str(max_lora_stat),
1856 waiting_lora_adapters=list(waiting_lora_adapters.keys()),
vllm/engine/metrics.py
+8
@@ -179,6 +179,12 @@ class Metrics:
179 labelnames=labelnames,
180 buckets=[1, 2, 5, 10, 20],
181 )
182 + self.histogram_max_tokens_request = self._histogram_cls(
183 + name="vllm:request_params_max_tokens",
184 + documentation="Histogram of the max_tokens request parameter.",
185 + labelnames=labelnames,
186 + buckets=build_1_2_5_buckets(max_model_len),
187 + )
188 self.counter_request_success = self._counter_cls(
189 name="vllm:request_success_total",
190 documentation="Count of successfully processed requests.",
@@ -547,6 +553,8 @@ class PrometheusStatLogger(StatLoggerBase):
553 self.metrics.histogram_num_generation_tokens_request,
554 stats.num_generation_tokens_requests)
555 self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
556 + self._log_histogram(self.metrics.histogram_max_tokens_request,
557 + stats.max_tokens_requests)
558
559 def _log_prometheus_interval(self, prompt_throughput: float,
560 generation_throughput: float) -> None:
vllm/engine/metrics_types.py
+1
@@ -53,6 +53,7 @@ class Stats:
53 num_prompt_tokens_requests: List[int]
54 num_generation_tokens_requests: List[int]
55 n_requests: List[int]
56 + max_tokens_requests: List[int]
57 finished_reason_requests: List[str]
58 waiting_lora_adapters: List[str]
59 running_lora_adapters: List[str]