A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1][Metrics] Hook up IterationStats for Prometheus metrics (#12478)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
Jan 28, 2025 at 16:38 UTC
3fd1fb63efb6c96f30237b12e2816b4f2c5323d0
3 files changed
+66
-12
tests/entrypoints/openai/test_metrics.py
+5
-2
@@ -105,8 +105,6 @@ EXPECTED_VALUES = {
105
@pytest.mark.asyncio
106
async def test_metrics_counts(server: RemoteOpenAIServer,
107
client: openai.AsyncClient, use_v1: bool):
108
- if use_v1:
109
- pytest.skip("Skipping test on vllm V1")
108
for _ in range(_NUM_REQUESTS):
109
# sending a request triggers the metrics to be logged.
110
await client.completions.create(
@@ -120,6 +118,9 @@ async def test_metrics_counts(server: RemoteOpenAIServer,
118
119
# Loop over all expected metric_families
120
for metric_family, suffix_values_list in EXPECTED_VALUES.items():
121
+ if use_v1 and metric_family not in EXPECTED_METRICS_V1:
122
+ continue
123
+
124
found_metric = False
125
126
# Check to see if the metric_family is found in the prom endpoint.
@@ -199,6 +200,8 @@ EXPECTED_METRICS = [
200
EXPECTED_METRICS_V1 = [
201
"vllm:num_requests_running",
202
"vllm:num_requests_waiting",
203
+ "vllm:prompt_tokens_total",
204
+ "vllm:generation_tokens_total",
205
]
206
207
vllm/v1/engine/async_llm.py
+2
-1
@@ -305,7 +305,8 @@ class AsyncLLM(EngineClient):
305
return
306
307
for logger in self.stat_loggers:
308
- logger.log(scheduler_stats=scheduler_stats)
308
+ logger.log(scheduler_stats=scheduler_stats,
309
+ iteration_stats=iteration_stats)
310
311
def encode(
312
self,
vllm/v1/metrics/loggers.py
+59
-9
@@ -1,11 +1,12 @@
1
import time
2
from abc import ABC, abstractmethod
3
-from typing import Dict
3
+from typing import Dict, List
4
5
+import numpy as np
6
import prometheus_client
7
8
from vllm.logger import init_logger
8
-from vllm.v1.metrics.stats import SchedulerStats
9
+from vllm.v1.metrics.stats import IterationStats, SchedulerStats
10
11
logger = init_logger(__name__)
12
@@ -15,27 +16,61 @@ _LOCAL_LOGGING_INTERVAL_SEC = 5.0
16
class StatLoggerBase(ABC):
17
18
@abstractmethod
18
- def log(self, scheduler_stats: SchedulerStats):
19
+ def log(self, scheduler_stats: SchedulerStats,
20
+ iteration_stats: IterationStats):
21
...
22
23
24
class LoggingStatLogger(StatLoggerBase):
25
26
def __init__(self):
25
- self.last_log_time = time.monotonic()
27
+ self._reset(time.monotonic())
28
27
- def log(self, scheduler_stats: SchedulerStats):
28
- """Log Stats to standard output."""
29
+ def _reset(self, now):
30
+ self.last_log_time = now
31
+
32
+ # Tracked stats over current local logging interval.
33
+ self.num_prompt_tokens: List[int] = []
34
+ self.num_generation_tokens: List[int] = []
35
36
+ def _local_interval_elapsed(self, now: float) -> bool:
37
# Log every _LOCAL_LOGGING_INTERVAL_SEC.
38
+ elapsed_time = now - self.last_log_time
39
+ return elapsed_time > _LOCAL_LOGGING_INTERVAL_SEC
40
+
41
+ def _track_iteration_stats(self, iteration_stats: IterationStats):
42
+ # Save tracked stats for token counters.
43
+ self.num_prompt_tokens.append(iteration_stats.num_prompt_tokens)
44
+ self.num_generation_tokens.append(
45
+ iteration_stats.num_generation_tokens)
46
+
47
+ def _get_throughput(self, tracked_stats: List[int], now: float) -> float:
48
+ # Compute summary metrics for tracked stats
49
+ return float(np.sum(tracked_stats) / (now - self.last_log_time))
50
+
51
+ def log(self, scheduler_stats: SchedulerStats,
52
+ iteration_stats: IterationStats):
53
+ """Log Stats to standard output."""
54
+
55
+ self._track_iteration_stats(iteration_stats)
56
+
57
now = time.monotonic()
32
- if now - self.last_log_time < _LOCAL_LOGGING_INTERVAL_SEC:
58
+ if not self._local_interval_elapsed(now):
59
return
34
- self.last_log_time = now
60
+
61
+ prompt_throughput = self._get_throughput(self.num_prompt_tokens, now)
62
+ generation_throughput = self._get_throughput(
63
+ self.num_generation_tokens, now)
64
+
65
+ self._reset(now)
66
67
# Format and print output.
68
logger.info(
69
+ "Avg prompt throughput: %.1f tokens/s, "
70
+ "Avg generation throughput: %.1f tokens/s, "
71
"Running: %d reqs, Waiting: %d reqs ",
72
+ prompt_throughput,
73
+ generation_throughput,
74
scheduler_stats.num_running_reqs,
75
scheduler_stats.num_waiting_reqs,
76
)
@@ -61,11 +96,26 @@ class PrometheusStatLogger(StatLoggerBase):
96
documentation="Number of requests waiting to be processed.",
97
labelnames=labelnames).labels(*labelvalues)
98
64
- def log(self, scheduler_stats: SchedulerStats):
99
+ self.counter_prompt_tokens = prometheus_client.Counter(
100
+ name="vllm:prompt_tokens_total",
101
+ documentation="Number of prefill tokens processed.",
102
+ labelnames=labelnames).labels(*labelvalues)
103
+
104
+ self.counter_generation_tokens = prometheus_client.Counter(
105
+ name="vllm:generation_tokens_total",
106
+ documentation="Number of generation tokens processed.",
107
+ labelnames=labelnames).labels(*labelvalues)
108
+
109
+ def log(self, scheduler_stats: SchedulerStats,
110
+ iteration_stats: IterationStats):
111
"""Log to prometheus."""
112
self.gauge_scheduler_running.set(scheduler_stats.num_running_reqs)
113
self.gauge_scheduler_waiting.set(scheduler_stats.num_waiting_reqs)
114
115
+ self.counter_prompt_tokens.inc(iteration_stats.num_prompt_tokens)
116
+ self.counter_generation_tokens.inc(
117
+ iteration_stats.num_generation_tokens)
118
+
119
@staticmethod
120
def _unregister_vllm_metrics():
121
# Unregister any existing vLLM collectors (for CI/CD