@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4078052f09
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][Log] Add max request concurrency log to V1 (#12569)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Jan 30, 2025 at 18:07 UTC 4078052f09f42f898b542e18d60d15a43db67a8b
1 file changed +4
vllm/v1/core/kv_cache_utils.py
+4
@@ -393,6 +393,10 @@ def _get_kv_cache_config_uniform_type(vllm_config: VllmConfig,
393 num_blocks = num_gpu_blocks_override
394
395 logger.info("# GPU blocks: %d", num_blocks)
396 + max_concurrency = (num_blocks * vllm_config.cache_config.block_size /
397 + vllm_config.model_config.max_model_len)
398 + logger.info("Maximum concurrency for %s tokens per request: %.2fx",
399 + vllm_config.model_config.max_model_len, max_concurrency)
400
401 per_layer_size = page_size * num_blocks
402