A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix incorrect kv cache metrics in grafana.json (#27133)
Signed-off-by: Fangping Shi <fangping_shi@apple.com> Co-authored-by: Fangping Shi <fangping_shi@apple.com>
fangpings committed
Oct 22, 2025 at 20:58 UTC
7e0941055fdf89bae93045683dd80542177f3241
3 files changed
+6
-6
examples/online_serving/dashboards/perses/performance_statistics.yaml
+1
-1
@@ -530,7 +530,7 @@ spec:
530
name: accelerators-thanos-querier-datasource
531
# Multiply by 100 so we can read it as a percentage without setting a unit (avoids CUE unit conflicts)
532
query: >
533
- 100 * avg(vllm:gpu_cache_usage_perc)
533
+ 100 * avg(vllm:kv_cache_usage_perc)
534
535
"18":
536
kind: Panel
examples/online_serving/dashboards/perses/query_statistics.yaml
+4
-4
@@ -98,7 +98,7 @@ spec:
98
kind: PrometheusTimeSeriesQuery
99
spec:
100
datasource: { kind: PrometheusDatasource, name: accelerators-thanos-querier-datasource }
101
- query: avg(vllm:gpu_cache_usage_perc{namespace="$NS",service="$SVC"}) or vector(0)
101
+ query: avg(vllm:kv_cache_usage_perc{namespace="$NS",service="$SVC"}) or vector(0)
102
minStep: "15s"
103
104
core_running_ts:
@@ -168,7 +168,7 @@ spec:
168
spec:
169
datasource: { kind: PrometheusDatasource, name: accelerators-thanos-querier-datasource }
170
# multiply by 100 to present percentage; omit format.unit to avoid schema conflicts
171
- query: (avg(vllm:gpu_cache_usage_perc{namespace="$NS",service="$SVC"}) * 100) or vector(0)
171
+ query: (avg(vllm:kv_cache_usage_perc{namespace="$NS",service="$SVC"}) * 100) or vector(0)
172
minStep: "15s"
173
174
core_kv_usage_pct_ts:
@@ -187,7 +187,7 @@ spec:
187
kind: PrometheusTimeSeriesQuery
188
spec:
189
datasource: { kind: PrometheusDatasource, name: accelerators-thanos-querier-datasource }
190
- query: (avg by (service) (vllm:gpu_cache_usage_perc{namespace="$NS",service="$SVC"}) * 100) or vector(0)
190
+ query: (avg by (service) (vllm:kv_cache_usage_perc{namespace="$NS",service="$SVC"}) * 100) or vector(0)
191
minStep: "15s"
192
193
# --- Per-Pod breakdowns (works on Simulator & Real) ---
@@ -246,7 +246,7 @@ spec:
246
spec:
247
datasource: { kind: PrometheusDatasource, name: accelerators-thanos-querier-datasource }
248
# if your exporter labels kv metric with pod (the sim does), this works; otherwise it will just return empty
249
- query: (avg by (pod) (vllm:gpu_cache_usage_perc{namespace="$NS",service="$SVC"}) * 100) or vector(0)
249
+ query: (avg by (pod) (vllm:kv_cache_usage_perc{namespace="$NS",service="$SVC"}) * 100) or vector(0)
250
minStep: "15s"
251
252
# --- Real vLLM only (zeros on simulator) ---
examples/online_serving/prometheus_grafana/grafana.json
+1
-1
@@ -852,7 +852,7 @@
852
"uid": "${DS_PROMETHEUS}"
853
},
854
"editorMode": "code",
855
- "expr": "vllm:gpu_cache_usage_perc{model_name=\"$model_name\"}",
855
+ "expr": "vllm:kv_cache_usage_perc{model_name=\"$model_name\"}",
856
"instant": false,
857
"legendFormat": "GPU Cache Usage",
858
"range": true,