A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[V1][Metrics] Add iteration_tokens_total histogram from V0 (#13288)
Mark McLoughlin committed
Feb 15, 2025 at 11:56 UTC
2ad1bc7afed42cdda02913ba437e7cc98b3d386d
3 files changed
+35
-8
tests/entrypoints/openai/test_metrics.py
+10
-3
@@ -96,9 +96,14 @@ EXPECTED_VALUES = {
96
[("_sum", _NUM_REQUESTS * _NUM_GENERATION_TOKENS_PER_REQUEST),
97
("_count", _NUM_REQUESTS)],
98
"vllm:request_params_n": [("_count", _NUM_REQUESTS)],
99
- "vllm:request_params_max_tokens":
100
- [("_sum", _NUM_REQUESTS * _NUM_GENERATION_TOKENS_PER_REQUEST),
101
- ("_count", _NUM_REQUESTS)],
99
+ "vllm:request_params_max_tokens": [
100
+ ("_sum", _NUM_REQUESTS * _NUM_GENERATION_TOKENS_PER_REQUEST),
101
+ ("_count", _NUM_REQUESTS)
102
+ ],
103
+ "vllm:iteration_tokens_total":
104
+ [("_sum", _NUM_REQUESTS *
105
+ (_NUM_PROMPT_TOKENS_PER_REQUEST + _NUM_GENERATION_TOKENS_PER_REQUEST)),
106
+ ("_count", _NUM_REQUESTS * _NUM_GENERATION_TOKENS_PER_REQUEST)],
107
"vllm:prompt_tokens": [("_total",
108
_NUM_REQUESTS * _NUM_PROMPT_TOKENS_PER_REQUEST)],
109
"vllm:generation_tokens": [
@@ -197,6 +202,7 @@ EXPECTED_METRICS = [
202
"vllm:request_params_max_tokens_sum",
203
"vllm:request_params_max_tokens_bucket",
204
"vllm:request_params_max_tokens_count",
205
+ "vllm:iteration_tokens_total",
206
"vllm:num_preemptions_total",
207
"vllm:prompt_tokens_total",
208
"vllm:generation_tokens_total",
@@ -223,6 +229,7 @@ EXPECTED_METRICS_V1 = [
229
"vllm:gpu_prefix_cache_hits",
230
"vllm:prompt_tokens_total",
231
"vllm:generation_tokens_total",
232
+ "vllm:iteration_tokens_total",
233
"vllm:request_success_total",
234
"vllm:request_prompt_tokens_sum",
235
"vllm:request_prompt_tokens_bucket",
vllm/v1/engine/async_llm.py
+1
-1
@@ -57,7 +57,7 @@ class AsyncLLM(EngineClient):
57
if self.log_stats:
58
self.stat_loggers.extend([
59
LoggingStatLogger(),
60
- PrometheusStatLogger(vllm_config.model_config),
60
+ PrometheusStatLogger(vllm_config),
61
])
62
63
# Tokenizer (+ ensure liveness if running in another process).
vllm/v1/metrics/loggers.py
+24
-4
@@ -7,7 +7,7 @@ from typing import Dict, List
7
import numpy as np
8
import prometheus_client
9
10
-from vllm.config import ModelConfig
10
+from vllm.config import VllmConfig
11
from vllm.logger import init_logger
12
from vllm.v1.core.kv_cache_utils import PrefixCachingMetrics
13
from vllm.v1.engine import FinishReason
@@ -92,13 +92,13 @@ class LoggingStatLogger(StatLoggerBase):
92
93
class PrometheusStatLogger(StatLoggerBase):
94
95
- def __init__(self, model_config: ModelConfig):
95
+ def __init__(self, vllm_config: VllmConfig):
96
self._unregister_vllm_metrics()
97
98
labelnames = ["model_name"]
99
- labelvalues = [model_config.served_model_name]
99
+ labelvalues = [vllm_config.model_config.served_model_name]
100
101
- max_model_len = model_config.max_model_len
101
+ max_model_len = vllm_config.model_config.max_model_len
102
103
self.gauge_scheduler_running = prometheus_client.Gauge(
104
name="vllm:num_requests_running",
@@ -162,6 +162,13 @@ class PrometheusStatLogger(StatLoggerBase):
162
buckets=build_1_2_5_buckets(max_model_len),
163
labelnames=labelnames).labels(*labelvalues)
164
165
+ self.histogram_iteration_tokens = \
166
+ prometheus_client.Histogram(
167
+ name="vllm:iteration_tokens_total",
168
+ documentation="Histogram of number of tokens per engine_step.",
169
+ buckets=build_cudagraph_buckets(vllm_config),
170
+ labelnames=labelnames).labels(*labelvalues)
171
+
172
self.histogram_time_to_first_token = \
173
prometheus_client.Histogram(
174
name="vllm:time_to_first_token_seconds",
@@ -237,6 +244,9 @@ class PrometheusStatLogger(StatLoggerBase):
244
self.counter_prompt_tokens.inc(iteration_stats.num_prompt_tokens)
245
self.counter_generation_tokens.inc(
246
iteration_stats.num_generation_tokens)
247
+ self.histogram_iteration_tokens.observe(
248
+ iteration_stats.num_prompt_tokens + \
249
+ iteration_stats.num_generation_tokens)
250
251
for finished_request in iteration_stats.finished_requests:
252
self.counter_request_success[finished_request.finish_reason].inc()
@@ -293,3 +303,13 @@ def build_1_2_5_buckets(max_value: int) -> List[int]:
303
[1, 2, 5, 10, 20, 50, 100]
304
"""
305
return build_buckets([1, 2, 5], max_value)
306
+
307
+
308
+def build_cudagraph_buckets(vllm_config: VllmConfig) -> List[int]:
309
+ if not vllm_config.model_config.enforce_eager:
310
+ buckets = vllm_config.compilation_config.\
311
+ cudagraph_capture_sizes.copy()
312
+ buckets.sort()
313
+ return buckets
314
+ else:
315
+ return [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096]