@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f17f1d4608
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][Metrics] Add GPU cache usage % gauge (#12561)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>

Mark McLoughlin committed Jan 30, 2025 at 02:31 UTC f17f1d46086692a2973fad94860a95799fbd8582
5 files changed +18 -2
tests/entrypoints/openai/test_metrics.py
+1
@@ -200,6 +200,7 @@ EXPECTED_METRICS = [
200 EXPECTED_METRICS_V1 = [
201 "vllm:num_requests_running",
202 "vllm:num_requests_waiting",
203 + "vllm:gpu_cache_usage_perc",
204 "vllm:prompt_tokens_total",
205 "vllm:generation_tokens_total",
206 "vllm:request_prompt_tokens_sum",
vllm/v1/core/kv_cache_manager.py
+5
@@ -69,6 +69,11 @@ class KVCacheManager:
69 # is finished.
70 self.req_to_blocks: Dict[str, List[KVCacheBlock]] = {}
71
72 + @property
73 + def usage(self) -> float:
74 + return 1.0 - (self.free_block_queue.num_free_blocks /
75 + self.num_gpu_blocks)
76 +
77 def get_computed_blocks(
78 self, request: Request) -> Tuple[List[KVCacheBlock], int]:
79 """Get the computed (cached) blocks for the request.
vllm/v1/core/scheduler.py
+1
@@ -544,6 +544,7 @@ class Scheduler:
544 return SchedulerStats(
545 num_running_reqs=len(self.running),
546 num_waiting_reqs=len(self.waiting),
547 + gpu_cache_usage=self.kv_cache_manager.usage,
548 )
549
550
vllm/v1/metrics/loggers.py
+10 -1
@@ -69,11 +69,13 @@ class LoggingStatLogger(StatLoggerBase):
69 logger.info(
70 "Avg prompt throughput: %.1f tokens/s, "
71 "Avg generation throughput: %.1f tokens/s, "
72 - "Running: %d reqs, Waiting: %d reqs ",
72 + "Running: %d reqs, Waiting: %d reqs "
73 + "GPU KV cache usage: %.1f%%.",
74 prompt_throughput,
75 generation_throughput,
76 scheduler_stats.num_running_reqs,
77 scheduler_stats.num_waiting_reqs,
78 + scheduler_stats.gpu_cache_usage * 100,
79 )
80
81
@@ -97,6 +99,11 @@ class PrometheusStatLogger(StatLoggerBase):
99 documentation="Number of requests waiting to be processed.",
100 labelnames=labelnames).labels(*labelvalues)
101
102 + self.gauge_gpu_cache_usage = prometheus_client.Gauge(
103 + name="vllm:gpu_cache_usage_perc",
104 + documentation="GPU KV-cache usage. 1 means 100 percent usage.",
105 + labelnames=labelnames).labels(*labelvalues)
106 +
107 self.counter_prompt_tokens = prometheus_client.Counter(
108 name="vllm:prompt_tokens_total",
109 documentation="Number of prefill tokens processed.",
@@ -147,6 +154,8 @@ class PrometheusStatLogger(StatLoggerBase):
154 self.gauge_scheduler_running.set(scheduler_stats.num_running_reqs)
155 self.gauge_scheduler_waiting.set(scheduler_stats.num_waiting_reqs)
156
157 + self.gauge_gpu_cache_usage.set(scheduler_stats.gpu_cache_usage)
158 +
159 self.counter_prompt_tokens.inc(iteration_stats.num_prompt_tokens)
160 self.counter_generation_tokens.inc(
161 iteration_stats.num_generation_tokens)
vllm/v1/metrics/stats.py
+1 -1
@@ -14,7 +14,7 @@ class SchedulerStats:
14 num_running_reqs: int = 0
15 num_waiting_reqs: int = 0
16
17 - # gpu_cache_usage: float = 0.0
17 + gpu_cache_usage: float = 0.0
18 # gpu_prefix_cache_hit_rate: float = 0.0
19
20