@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0b5b5d767e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Frontend] Log the maximum supported concurrency (#8831)

AlpinDale committed Oct 9, 2024 at 07:03 UTC 0b5b5d767e7fdc0b1070b37319de749e46a4d42a
2 files changed +8
vllm/executor/distributed_gpu_executor.py
+4
@@ -56,6 +56,10 @@ class DistributedGPUExecutor(GPUExecutor):
56 # have GPUs.
57 logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
58 num_cpu_blocks)
59 + max_concurrency = (num_gpu_blocks * self.cache_config.block_size /
60 + self.model_config.max_model_len)
61 + logger.info("Maximum concurrency for %s tokens per request: %.2fx",
62 + self.model_config.max_model_len, max_concurrency)
63
64 self.cache_config.num_gpu_blocks = num_gpu_blocks
65 self.cache_config.num_cpu_blocks = num_cpu_blocks
vllm/executor/gpu_executor.py
+4
@@ -121,6 +121,10 @@ class GPUExecutor(ExecutorBase):
121 # remains to abstract away the device for non-GPU configurations.
122 logger.info("# GPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
123 num_cpu_blocks)
124 + max_concurrency = (num_gpu_blocks * self.cache_config.block_size /
125 + self.model_config.max_model_len)
126 + logger.info("Maximum concurrency for %s tokens per request: %.2fx",
127 + self.model_config.max_model_len, max_concurrency)
128
129 self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
130