A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[V1][Minor] Print KV cache size in token counts (#13596)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Feb 20, 2025 at 09:24 UTC
d3ea50113c08bdd3c5cfda42ec6ecbc72328d7d1
1 file changed
+6
-4
vllm/v1/core/kv_cache_utils.py
+6
-4
@@ -519,11 +519,13 @@ def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
519
"num_gpu_blocks_override=%d", num_blocks, num_gpu_blocks_override)
520
num_blocks = num_gpu_blocks_override
521
522
- logger.info("# GPU blocks: %d", num_blocks)
523
- max_concurrency = (num_blocks * vllm_config.cache_config.block_size /
524
- vllm_config.model_config.max_model_len)
522
+ num_tokens = num_blocks * vllm_config.cache_config.block_size
523
+ num_tokens_str = f"{num_tokens:,}"
524
+ logger.info("GPU KV cache size: %s tokens", num_tokens_str)
525
+ max_model_len_str = f"{vllm_config.model_config.max_model_len:,}"
526
+ max_concurrency = num_tokens / vllm_config.model_config.max_model_len
527
logger.info("Maximum concurrency for %s tokens per request: %.2fx",
526
- vllm_config.model_config.max_model_len, max_concurrency)
528
+ max_model_len_str, max_concurrency)
529
530
per_layer_size = page_size * num_blocks
531