@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3fd1fb63ef
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][Metrics] Hook up IterationStats for Prometheus metrics (#12478)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>

Mark McLoughlin committed Jan 28, 2025 at 16:38 UTC 3fd1fb63efb6c96f30237b12e2816b4f2c5323d0
3 files changed +66 -12
tests/entrypoints/openai/test_metrics.py
+5 -2
@@ -105,8 +105,6 @@ EXPECTED_VALUES = {
105 @pytest.mark.asyncio
106 async def test_metrics_counts(server: RemoteOpenAIServer,
107 client: openai.AsyncClient, use_v1: bool):
108 - if use_v1:
109 - pytest.skip("Skipping test on vllm V1")
108 for _ in range(_NUM_REQUESTS):
109 # sending a request triggers the metrics to be logged.
110 await client.completions.create(
@@ -120,6 +118,9 @@ async def test_metrics_counts(server: RemoteOpenAIServer,
118
119 # Loop over all expected metric_families
120 for metric_family, suffix_values_list in EXPECTED_VALUES.items():
121 + if use_v1 and metric_family not in EXPECTED_METRICS_V1:
122 + continue
123 +
124 found_metric = False
125
126 # Check to see if the metric_family is found in the prom endpoint.
@@ -199,6 +200,8 @@ EXPECTED_METRICS = [
200 EXPECTED_METRICS_V1 = [
201 "vllm:num_requests_running",
202 "vllm:num_requests_waiting",
203 + "vllm:prompt_tokens_total",
204 + "vllm:generation_tokens_total",
205 ]
206
207
vllm/v1/engine/async_llm.py
+2 -1
@@ -305,7 +305,8 @@ class AsyncLLM(EngineClient):
305 return
306
307 for logger in self.stat_loggers:
308 - logger.log(scheduler_stats=scheduler_stats)
308 + logger.log(scheduler_stats=scheduler_stats,
309 + iteration_stats=iteration_stats)
310
311 def encode(
312 self,
vllm/v1/metrics/loggers.py
+59 -9
@@ -1,11 +1,12 @@
1 import time
2 from abc import ABC, abstractmethod
3 -from typing import Dict
3 +from typing import Dict, List
4
5 +import numpy as np
6 import prometheus_client
7
8 from vllm.logger import init_logger
8 -from vllm.v1.metrics.stats import SchedulerStats
9 +from vllm.v1.metrics.stats import IterationStats, SchedulerStats
10
11 logger = init_logger(__name__)
12
@@ -15,27 +16,61 @@ _LOCAL_LOGGING_INTERVAL_SEC = 5.0
16 class StatLoggerBase(ABC):
17
18 @abstractmethod
18 - def log(self, scheduler_stats: SchedulerStats):
19 + def log(self, scheduler_stats: SchedulerStats,
20 + iteration_stats: IterationStats):
21 ...
22
23
24 class LoggingStatLogger(StatLoggerBase):
25
26 def __init__(self):
25 - self.last_log_time = time.monotonic()
27 + self._reset(time.monotonic())
28
27 - def log(self, scheduler_stats: SchedulerStats):
28 - """Log Stats to standard output."""
29 + def _reset(self, now):
30 + self.last_log_time = now
31 +
32 + # Tracked stats over current local logging interval.
33 + self.num_prompt_tokens: List[int] = []
34 + self.num_generation_tokens: List[int] = []
35
36 + def _local_interval_elapsed(self, now: float) -> bool:
37 # Log every _LOCAL_LOGGING_INTERVAL_SEC.
38 + elapsed_time = now - self.last_log_time
39 + return elapsed_time > _LOCAL_LOGGING_INTERVAL_SEC
40 +
41 + def _track_iteration_stats(self, iteration_stats: IterationStats):
42 + # Save tracked stats for token counters.
43 + self.num_prompt_tokens.append(iteration_stats.num_prompt_tokens)
44 + self.num_generation_tokens.append(
45 + iteration_stats.num_generation_tokens)
46 +
47 + def _get_throughput(self, tracked_stats: List[int], now: float) -> float:
48 + # Compute summary metrics for tracked stats
49 + return float(np.sum(tracked_stats) / (now - self.last_log_time))
50 +
51 + def log(self, scheduler_stats: SchedulerStats,
52 + iteration_stats: IterationStats):
53 + """Log Stats to standard output."""
54 +
55 + self._track_iteration_stats(iteration_stats)
56 +
57 now = time.monotonic()
32 - if now - self.last_log_time < _LOCAL_LOGGING_INTERVAL_SEC:
58 + if not self._local_interval_elapsed(now):
59 return
34 - self.last_log_time = now
60 +
61 + prompt_throughput = self._get_throughput(self.num_prompt_tokens, now)
62 + generation_throughput = self._get_throughput(
63 + self.num_generation_tokens, now)
64 +
65 + self._reset(now)
66
67 # Format and print output.
68 logger.info(
69 + "Avg prompt throughput: %.1f tokens/s, "
70 + "Avg generation throughput: %.1f tokens/s, "
71 "Running: %d reqs, Waiting: %d reqs ",
72 + prompt_throughput,
73 + generation_throughput,
74 scheduler_stats.num_running_reqs,
75 scheduler_stats.num_waiting_reqs,
76 )
@@ -61,11 +96,26 @@ class PrometheusStatLogger(StatLoggerBase):
96 documentation="Number of requests waiting to be processed.",
97 labelnames=labelnames).labels(*labelvalues)
98
64 - def log(self, scheduler_stats: SchedulerStats):
99 + self.counter_prompt_tokens = prometheus_client.Counter(
100 + name="vllm:prompt_tokens_total",
101 + documentation="Number of prefill tokens processed.",
102 + labelnames=labelnames).labels(*labelvalues)
103 +
104 + self.counter_generation_tokens = prometheus_client.Counter(
105 + name="vllm:generation_tokens_total",
106 + documentation="Number of generation tokens processed.",
107 + labelnames=labelnames).labels(*labelvalues)
108 +
109 + def log(self, scheduler_stats: SchedulerStats,
110 + iteration_stats: IterationStats):
111 """Log to prometheus."""
112 self.gauge_scheduler_running.set(scheduler_stats.num_running_reqs)
113 self.gauge_scheduler_waiting.set(scheduler_stats.num_waiting_reqs)
114
115 + self.counter_prompt_tokens.inc(iteration_stats.num_prompt_tokens)
116 + self.counter_generation_tokens.inc(
117 + iteration_stats.num_generation_tokens)
118 +
119 @staticmethod
120 def _unregister_vllm_metrics():
121 # Unregister any existing vLLM collectors (for CI/CD