A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[V1][Metrics] Deprecate metrics with gpu_ prefix for non GPU specific metrics. (#18354)
Signed-off-by: Saheli Bhattacharjee <saheli@krai.ai>
Saheli Bhattacharjee committed
Jun 14, 2025 at 04:07 UTC
d1e34cc9ac2a1fb466283225086b8bed1f7c99a9
3 files changed
+41
-8
vllm/v1/core/sched/scheduler.py
+1
-1
@@ -947,7 +947,7 @@ class Scheduler(SchedulerInterface):
947
return SchedulerStats(
948
num_running_reqs=len(self.running),
949
num_waiting_reqs=len(self.waiting),
950
- gpu_cache_usage=self.kv_cache_manager.usage,
950
+ kv_cache_usage=self.kv_cache_manager.usage,
951
prefix_cache_stats=prefix_cache_stats,
952
spec_decoding_stats=spec_decoding_stats,
953
)
vllm/v1/metrics/loggers.py
+39
-6
@@ -127,7 +127,7 @@ class LoggingStatLogger(StatLoggerBase):
127
generation_throughput,
128
scheduler_stats.num_running_reqs,
129
scheduler_stats.num_waiting_reqs,
130
- scheduler_stats.gpu_cache_usage * 100,
130
+ scheduler_stats.kv_cache_usage * 100,
131
self.prefix_caching_metrics.hit_rate * 100,
132
)
133
self.spec_decoding_logging.log(log_fn=log_fn)
@@ -185,22 +185,49 @@ class PrometheusStatLogger(StatLoggerBase):
185
#
186
# GPU cache
187
#
188
+ # Deprecated in 0.9 - Renamed as vllm:kv_cache_usage_perc
189
+ # TODO: in 0.10, only enable if show_hidden_metrics=True
190
self.gauge_gpu_cache_usage = self._gauge_cls(
191
name="vllm:gpu_cache_usage_perc",
190
- documentation="GPU KV-cache usage. 1 means 100 percent usage.",
192
+ documentation=(
193
+ "GPU KV-cache usage. 1 means 100 percent usage."
194
+ "DEPRECATED: Use vllm:kv_cache_usage_perc instead."),
195
multiprocess_mode="mostrecent",
196
labelnames=labelnames).labels(*labelvalues)
197
198
+ # Deprecated in 0.9 - Renamed as vllm:prefix_cache_queries
199
+ # TODO: in 0.10, only enable if show_hidden_metrics=True
200
self.counter_gpu_prefix_cache_queries = self._counter_cls(
201
name="vllm:gpu_prefix_cache_queries",
202
documentation=
197
- "GPU prefix cache queries, in terms of number of queried tokens.",
203
+ ("GPU prefix cache queries, in terms of number of queried tokens."
204
+ "DEPRECATED: Use vllm:prefix_cache_queries instead."),
205
labelnames=labelnames).labels(*labelvalues)
206
207
+ # Deprecated in 0.9 - Renamed as vllm:prefix_cache_hits
208
+ # TODO: in 0.10, only enable if show_hidden_metrics=True
209
self.counter_gpu_prefix_cache_hits = self._counter_cls(
210
name="vllm:gpu_prefix_cache_hits",
202
- documentation=
203
- "GPU prefix cache hits, in terms of number of cached tokens.",
211
+ documentation=(
212
+ "GPU prefix cache hits, in terms of number of cached tokens."
213
+ "DEPRECATED: Use vllm:prefix_cache_hits instead."),
214
+ labelnames=labelnames).labels(*labelvalues)
215
+
216
+ self.gauge_kv_cache_usage = self._gauge_cls(
217
+ name="vllm:kv_cache_usage_perc",
218
+ documentation="KV-cache usage. 1 means 100 percent usage.",
219
+ labelnames=labelnames).labels(*labelvalues)
220
+
221
+ self.counter_prefix_cache_queries = self._counter_cls(
222
+ name="vllm:prefix_cache_queries",
223
+ documentation=(
224
+ "Prefix cache queries, in terms of number of queried tokens."),
225
+ labelnames=labelnames).labels(*labelvalues)
226
+
227
+ self.counter_prefix_cache_hits = self._counter_cls(
228
+ name="vllm:prefix_cache_hits",
229
+ documentation=(
230
+ "Prefix cache hits, in terms of number of cached tokens."),
231
labelnames=labelnames).labels(*labelvalues)
232
233
#
@@ -400,13 +427,19 @@ class PrometheusStatLogger(StatLoggerBase):
427
self.gauge_scheduler_running.set(scheduler_stats.num_running_reqs)
428
self.gauge_scheduler_waiting.set(scheduler_stats.num_waiting_reqs)
429
403
- self.gauge_gpu_cache_usage.set(scheduler_stats.gpu_cache_usage)
430
+ self.gauge_gpu_cache_usage.set(scheduler_stats.kv_cache_usage)
431
+ self.gauge_kv_cache_usage.set(scheduler_stats.kv_cache_usage)
432
433
self.counter_gpu_prefix_cache_queries.inc(
434
scheduler_stats.prefix_cache_stats.queries)
435
self.counter_gpu_prefix_cache_hits.inc(
436
scheduler_stats.prefix_cache_stats.hits)
437
438
+ self.counter_prefix_cache_queries.inc(
439
+ scheduler_stats.prefix_cache_stats.queries)
440
+ self.counter_prefix_cache_hits.inc(
441
+ scheduler_stats.prefix_cache_stats.hits)
442
+
443
if scheduler_stats.spec_decoding_stats is not None:
444
self.spec_decoding_prom.observe(
445
scheduler_stats.spec_decoding_stats)
vllm/v1/metrics/stats.py
+1
-1
@@ -33,7 +33,7 @@ class SchedulerStats:
33
num_running_reqs: int = 0
34
num_waiting_reqs: int = 0
35
36
- gpu_cache_usage: float = 0.0
36
+ kv_cache_usage: float = 0.0
37
38
prefix_cache_stats: PrefixCacheStats = field(
39
default_factory=PrefixCacheStats)