A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1][Metrics] Add GPU cache usage % gauge (#12561)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
Jan 30, 2025 at 02:31 UTC
f17f1d46086692a2973fad94860a95799fbd8582
5 files changed
+18
-2
tests/entrypoints/openai/test_metrics.py
+1
@@ -200,6 +200,7 @@ EXPECTED_METRICS = [
200
EXPECTED_METRICS_V1 = [
201
"vllm:num_requests_running",
202
"vllm:num_requests_waiting",
203
+ "vllm:gpu_cache_usage_perc",
204
"vllm:prompt_tokens_total",
205
"vllm:generation_tokens_total",
206
"vllm:request_prompt_tokens_sum",
vllm/v1/core/kv_cache_manager.py
+5
@@ -69,6 +69,11 @@ class KVCacheManager:
69
# is finished.
70
self.req_to_blocks: Dict[str, List[KVCacheBlock]] = {}
71
72
+ @property
73
+ def usage(self) -> float:
74
+ return 1.0 - (self.free_block_queue.num_free_blocks /
75
+ self.num_gpu_blocks)
76
+
77
def get_computed_blocks(
78
self, request: Request) -> Tuple[List[KVCacheBlock], int]:
79
"""Get the computed (cached) blocks for the request.
vllm/v1/core/scheduler.py
+1
@@ -544,6 +544,7 @@ class Scheduler:
544
return SchedulerStats(
545
num_running_reqs=len(self.running),
546
num_waiting_reqs=len(self.waiting),
547
+ gpu_cache_usage=self.kv_cache_manager.usage,
548
)
549
550
vllm/v1/metrics/loggers.py
+10
-1
@@ -69,11 +69,13 @@ class LoggingStatLogger(StatLoggerBase):
69
logger.info(
70
"Avg prompt throughput: %.1f tokens/s, "
71
"Avg generation throughput: %.1f tokens/s, "
72
- "Running: %d reqs, Waiting: %d reqs ",
72
+ "Running: %d reqs, Waiting: %d reqs "
73
+ "GPU KV cache usage: %.1f%%.",
74
prompt_throughput,
75
generation_throughput,
76
scheduler_stats.num_running_reqs,
77
scheduler_stats.num_waiting_reqs,
78
+ scheduler_stats.gpu_cache_usage * 100,
79
)
80
81
@@ -97,6 +99,11 @@ class PrometheusStatLogger(StatLoggerBase):
99
documentation="Number of requests waiting to be processed.",
100
labelnames=labelnames).labels(*labelvalues)
101
102
+ self.gauge_gpu_cache_usage = prometheus_client.Gauge(
103
+ name="vllm:gpu_cache_usage_perc",
104
+ documentation="GPU KV-cache usage. 1 means 100 percent usage.",
105
+ labelnames=labelnames).labels(*labelvalues)
106
+
107
self.counter_prompt_tokens = prometheus_client.Counter(
108
name="vllm:prompt_tokens_total",
109
documentation="Number of prefill tokens processed.",
@@ -147,6 +154,8 @@ class PrometheusStatLogger(StatLoggerBase):
154
self.gauge_scheduler_running.set(scheduler_stats.num_running_reqs)
155
self.gauge_scheduler_waiting.set(scheduler_stats.num_waiting_reqs)
156
157
+ self.gauge_gpu_cache_usage.set(scheduler_stats.gpu_cache_usage)
158
+
159
self.counter_prompt_tokens.inc(iteration_stats.num_prompt_tokens)
160
self.counter_generation_tokens.inc(
161
iteration_stats.num_generation_tokens)
vllm/v1/metrics/stats.py
+1
-1
@@ -14,7 +14,7 @@ class SchedulerStats:
14
num_running_reqs: int = 0
15
num_waiting_reqs: int = 0
16
17
- # gpu_cache_usage: float = 0.0
17
+ gpu_cache_usage: float = 0.0
18
# gpu_prefix_cache_hit_rate: float = 0.0
19
20