[Bugfix][Core] Log KV cache capacity after block-size resolution (#50462)
Signed-off-by: Tanmay Dixit <22117787+tandixit95@users.noreply.github.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Tanmay Dixit <22117787+tandixit95@users.noreply.github.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Tanmay Dixit committed
Aug 4, 2026 at 08:53 UTC
5f2ee2fa8c59d9f9cddd07c79806e6418b0d841a
2 files changed
+19
-23
vllm/v1/core/kv_cache_utils.py
+17
-17
@@ -1853,6 +1853,23 @@ def get_kv_cache_capacity(
1853
return int(max_concurrency * max_model_len), max_concurrency
1854
1855
1856
+def update_kv_cache_capacity(
1857
+ vllm_config: VllmConfig, kv_cache_config: KVCacheConfig
1858
+) -> None:
1859
+ """Store and log the resolved KV cache capacity."""
1860
+ num_tokens, max_concurrency = get_kv_cache_capacity(vllm_config, kv_cache_config)
1861
+ vllm_config.cache_config.kv_cache_size_tokens = num_tokens
1862
+ vllm_config.cache_config.kv_cache_max_concurrency = max_concurrency
1863
+ max_model_len = vllm_config.model_config.max_model_len
1864
+ logger.info_once(
1865
+ "GPU KV cache size: %s tokens, "
1866
+ "Maximum concurrency for %s tokens per request: %.2fx",
1867
+ f"{num_tokens:,}",
1868
+ f"{max_model_len:,}",
1869
+ max_concurrency,
1870
+ )
1871
+
1872
+
1873
def _max_memory_usage_bytes_from_groups(
1874
vllm_config: VllmConfig,
1875
kv_cache_groups: list[KVCacheGroupSpec],
@@ -2188,23 +2205,6 @@ def get_kv_cache_configs(
2205
assert tensor.size % num_blocks_old == 0
2206
tensor.size = tensor.size // num_blocks_old * min_num_blocks
2207
2191
- if len(kv_cache_config.kv_cache_groups) > 0:
2192
- max_model_len = vllm_config.model_config.max_model_len
2193
- # GPU KV cache size in tokens = max_concurrency * max_model_len:
2194
- # the total tokens of context the pool can hold at peak
2195
- # utilization. Sourcing this from the concurrency calculation
2196
- # handles hybrid layouts correctly.
2197
- num_tokens, max_concurrency = get_kv_cache_capacity(
2198
- vllm_config, kv_cache_config
2199
- )
2200
-
2201
- logger.info_once("GPU KV cache size: %s tokens", f"{num_tokens:,}")
2202
- logger.info_once(
2203
- "Maximum concurrency for %s tokens per request: %.2fx",
2204
- f"{max_model_len:,}",
2205
- max_concurrency,
2206
- )
2207
-
2208
return kv_cache_configs
2209
2210
vllm/v1/engine/core.py
+2
-6
@@ -46,11 +46,11 @@ from vllm.utils.system_utils import decorate_logs, set_process_title
46
from vllm.v1.core.kv_cache_utils import (
47
BlockHash,
48
generate_scheduler_kv_cache_config,
49
- get_kv_cache_capacity,
49
get_kv_cache_configs,
50
get_request_block_hasher,
51
init_none_hash,
52
resolve_kv_cache_block_sizes,
53
+ update_kv_cache_capacity,
54
)
55
from vllm.v1.core.sched.interface import PauseState, SchedulerInterface
56
from vllm.v1.core.sched.output import SchedulerOutput
@@ -319,11 +319,7 @@ class EngineCore:
319
vllm_config.cache_config.block_size = min(
320
g.kv_cache_spec.block_size for g in kv_cache_groups
321
)
322
- num_tokens, max_concurrency = get_kv_cache_capacity(
323
- vllm_config, scheduler_kv_cache_config
324
- )
325
- vllm_config.cache_config.kv_cache_size_tokens = num_tokens
326
- vllm_config.cache_config.kv_cache_max_concurrency = max_concurrency
322
+ update_kv_cache_capacity(vllm_config, scheduler_kv_cache_config)
323
324
vllm_config.validate_block_size()
325