A high-throughput and memory-efficient inference and serving engine for LLMs
[Metrics] Add group-aware KV cache capacity to vllm:cache_config_info (#42206)
The startup log already reports the correct group-aware KV cache capacity for hybrid models, but Prometheus did not expose matching info in 'vllm:cache_config_info`. This PR adds kv_cache_size_tokens and kv_cache_max_concurrency. Signed-off-by: Ethan Feng <ethan.fengch@gmail.com>
Ethan Feng committed
Jun 12, 2026 at 19:49 UTC
b7f9b6ab271faa621f4cc438fd5ea7ecaf72db8e
7 files changed
+75
-26
tests/entrypoints/serve/instrumentator/test_metrics.py
+11
@@ -289,6 +289,17 @@ async def test_metrics_exist(
289
continue
290
assert metric in response.text
291
292
+ cache_config_samples = [
293
+ sample
294
+ for family in text_string_to_metric_families(response.text)
295
+ if family.name == "vllm:cache_config_info"
296
+ for sample in family.samples
297
+ ]
298
+ assert cache_config_samples
299
+ for sample in cache_config_samples:
300
+ assert sample.labels.get("kv_cache_size_tokens") not in (None, "None", "")
301
+ assert sample.labels.get("kv_cache_max_concurrency") not in (None, "None", "")
302
+
303
304
@pytest.mark.asyncio
305
async def test_abort_metrics_reset(
tests/v1/core/test_kv_cache_utils.py
+6
@@ -28,6 +28,7 @@ from vllm.v1.core.kv_cache_utils import (
28
estimate_max_model_len,
29
generate_block_hash_extra_keys,
30
generate_scheduler_kv_cache_config,
31
+ get_kv_cache_capacity,
32
get_kv_cache_configs,
33
get_max_concurrency_for_kv_cache_config,
34
get_request_block_hasher,
@@ -1459,6 +1460,11 @@ def test_get_max_concurrency_for_kv_cache_config():
1460
vllm_config, kv_cache_config_hybrid_model
1461
)
1462
assert max_concurrency_hybrid_model == 3
1463
+ num_tokens, max_concurrency = get_kv_cache_capacity(
1464
+ vllm_config, kv_cache_config_hybrid_model
1465
+ )
1466
+ assert num_tokens == max_concurrency_hybrid_model * max_model_len
1467
+ assert max_concurrency == max_concurrency_hybrid_model
1468
1469
1470
def test_allocate_with_lookahead():
vllm/config/cache.py
+10
@@ -146,6 +146,14 @@ class CacheConfig:
146
num_cpu_blocks: int | None = field(default=None, init=False)
147
"""The number of blocks to allocate for CPU memory."""
148
149
+ # Set after KV cache initialization.
150
+ kv_cache_size_tokens: int | None = field(default=None, init=False)
151
+ """Per-DP-engine KV cache capacity in tokens (group-aware). Uses
152
+ group-aware capacity since num_gpu_blocks * block_size can be wrong
153
+ for hybrid models where requests occupy multiple KV cache groups."""
154
+ kv_cache_max_concurrency: float | None = field(default=None, init=False)
155
+ """Per-DP-engine maximum concurrency at max_model_len tokens."""
156
+
157
kv_sharing_fast_prefill: bool = False
158
"""This feature is work in progress and no prefill optimization takes place
159
with this flag enabled currently.
@@ -204,6 +212,8 @@ class CacheConfig:
212
# Post-init/derived counters
213
"num_gpu_blocks",
214
"num_cpu_blocks",
215
+ "kv_cache_size_tokens",
216
+ "kv_cache_max_concurrency",
217
# WIP feature toggle not impacting compiled graph shape
218
"kv_sharing_fast_prefill",
219
}
vllm/v1/core/kv_cache_utils.py
+19
-24
@@ -1717,36 +1717,17 @@ def generate_scheduler_kv_cache_config(
1717
return cfg
1718
1719
1720
-def _report_kv_cache_config(
1720
+def get_kv_cache_capacity(
1721
vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
1722
-) -> None:
1722
+) -> tuple[int, float]:
1723
"""
1724
- Log resolved KV cache configuration.
1725
-
1726
- Args:
1727
- vllm_config: The global VllmConfig
1728
- kv_cache_config: The resolved KV cache configuration
1724
+ Get the group-aware KV cache token capacity and max concurrency.
1725
"""
1726
max_model_len = vllm_config.model_config.max_model_len
1727
max_concurrency = get_max_concurrency_for_kv_cache_config(
1728
vllm_config, kv_cache_config
1729
)
1734
-
1735
- # GPU KV cache size in tokens = max_concurrency * max_model_len: the total
1736
- # tokens of context the pool can hold at peak utilization. Sourcing this
1737
- # from the concurrency calculation handles hybrid layouts correctly: SWA /
1738
- # chunked-local groups have a per-request block count that's capped by
1739
- # their window, so a naive `num_blocks // num_groups * block_size` formula
1740
- # underestimates capacity for these models. DCP/PCP sharding is already
1741
- # accounted for in each spec's `max_memory_usage_bytes`.
1742
- num_tokens = int(max_concurrency * max_model_len)
1743
-
1744
- logger.info_once("GPU KV cache size: %s tokens", f"{num_tokens:,}")
1745
- logger.info_once(
1746
- "Maximum concurrency for %s tokens per request: %.2fx",
1747
- f"{max_model_len:,}",
1748
- max_concurrency,
1749
- )
1730
+ return int(max_concurrency * max_model_len), max_concurrency
1731
1732
1733
def _max_memory_usage_bytes_from_groups(
@@ -2085,7 +2066,21 @@ def get_kv_cache_configs(
2066
tensor.size = tensor.size // num_blocks_old * min_num_blocks
2067
2068
if len(kv_cache_config.kv_cache_groups) > 0:
2088
- _report_kv_cache_config(vllm_config, kv_cache_config)
2069
+ max_model_len = vllm_config.model_config.max_model_len
2070
+ # GPU KV cache size in tokens = max_concurrency * max_model_len:
2071
+ # the total tokens of context the pool can hold at peak
2072
+ # utilization. Sourcing this from the concurrency calculation
2073
+ # handles hybrid layouts correctly.
2074
+ num_tokens, max_concurrency = get_kv_cache_capacity(
2075
+ vllm_config, kv_cache_config
2076
+ )
2077
+
2078
+ logger.info_once("GPU KV cache size: %s tokens", f"{num_tokens:,}")
2079
+ logger.info_once(
2080
+ "Maximum concurrency for %s tokens per request: %.2fx",
2081
+ f"{max_model_len:,}",
2082
+ max_concurrency,
2083
+ )
2084
2085
return kv_cache_configs
2086
vllm/v1/engine/__init__.py
+3
@@ -78,6 +78,9 @@ class EngineCoreReadyResponse:
78
dp_stats_address: str | None
79
dtype: str
80
vllm_version: str
81
+ # KV cache capacity (None for encoder-only/attention-free models).
82
+ kv_cache_size_tokens: int | None = None
83
+ kv_cache_max_concurrency: float | None = None
84
85
86
class EngineCoreRequest(
vllm/v1/engine/core.py
+12
@@ -45,6 +45,7 @@ from vllm.utils.system_utils import decorate_logs, set_process_title
45
from vllm.v1.core.kv_cache_utils import (
46
BlockHash,
47
generate_scheduler_kv_cache_config,
48
+ get_kv_cache_capacity,
49
get_kv_cache_configs,
50
get_request_block_hasher,
51
init_none_hash,
@@ -286,6 +287,11 @@ class EngineCore:
287
vllm_config.cache_config.block_size = min(
288
g.kv_cache_spec.block_size for g in kv_cache_groups
289
)
290
+ num_tokens, max_concurrency = get_kv_cache_capacity(
291
+ vllm_config, scheduler_kv_cache_config
292
+ )
293
+ vllm_config.cache_config.kv_cache_size_tokens = num_tokens
294
+ vllm_config.cache_config.kv_cache_max_concurrency = max_concurrency
295
296
vllm_config.validate_block_size()
297
@@ -1494,6 +1500,12 @@ class EngineCoreProc(EngineCore):
1500
dp_stats_address=self.frontend_stats_publish_address,
1501
dtype=str(self.vllm_config.model_config.dtype).removeprefix("torch."),
1502
vllm_version=VLLM_VERSION,
1503
+ kv_cache_size_tokens=(
1504
+ self.vllm_config.cache_config.kv_cache_size_tokens
1505
+ ),
1506
+ kv_cache_max_concurrency=(
1507
+ self.vllm_config.cache_config.kv_cache_max_concurrency
1508
+ ),
1509
)
1510
ready_payload = msgspec.msgpack.encode(ready_response)
1511
for input_socket in input_sockets:
vllm/v1/engine/core_client.py
+14
-2
@@ -720,14 +720,26 @@ class MPClient(EngineCoreClient):
720
)
721
722
# Setup KV cache config with initialization state from
723
- # engine core process. Sum values from all engines in DP case.
723
+ # engine core process. Sum num_gpu_blocks from all engines in DP case.
724
num_gpu_blocks = vllm_config.cache_config.num_gpu_blocks or 0
725
num_gpu_blocks += response.num_gpu_blocks
726
vllm_config.cache_config.num_gpu_blocks = num_gpu_blocks
727
728
# Sync block_size: may be enlarged by _align_hybrid_block_size in the
729
# worker for hybrid Mamba models.
730
- vllm_config.cache_config.block_size = response.block_size
730
+ cache_config = vllm_config.cache_config
731
+ cache_config.block_size = response.block_size
732
+ # Keep these as per-engine cache_config_info values; do not sum across DP.
733
+ cache_config.kv_cache_size_tokens = (
734
+ getattr(cache_config, "kv_cache_size_tokens", None)
735
+ if getattr(cache_config, "kv_cache_size_tokens", None) is not None
736
+ else response.kv_cache_size_tokens
737
+ )
738
+ cache_config.kv_cache_max_concurrency = (
739
+ getattr(cache_config, "kv_cache_max_concurrency", None)
740
+ if getattr(cache_config, "kv_cache_max_concurrency", None) is not None
741
+ else response.kv_cache_max_concurrency
742
+ )
743
744
# In external DP LB mode, the coordinator address that the
745
# front-end procs connect to is obtained by each engine via it's