A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1][Log] Add max request concurrency log to V1 (#12569)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Jan 30, 2025 at 18:07 UTC
4078052f09f42f898b542e18d60d15a43db67a8b
1 file changed
+4
vllm/v1/core/kv_cache_utils.py
+4
@@ -393,6 +393,10 @@ def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
393
num_blocks = num_gpu_blocks_override
394
395
logger.info("# GPU blocks: %d", num_blocks)
396
+ max_concurrency = (num_blocks * vllm_config.cache_config.block_size /
397
+ vllm_config.model_config.max_model_len)
398
+ logger.info("Maximum concurrency for %s tokens per request: %.2fx",
399
+ vllm_config.model_config.max_model_len, max_concurrency)
400
401
per_layer_size = page_size * num_blocks
402