A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Misc] Fix estimated max model len msg (#18966)
Signed-off-by: Yong Hoon Shin <yhshin@meta.com>
Yong Hoon Shin committed
May 31, 2025 at 01:43 UTC
1e123529d7df1ff8f868b19aeced6a64e67bd618
1 file changed
+5
-4
vllm/v1/core/kv_cache_utils.py
+5
-4
@@ -544,16 +544,17 @@ def check_enough_kv_cache_memory(vllm_config: VllmConfig,
544
available_memory)
545
estimated_msg = ""
546
if estimated_max_len > 0:
547
- estimated_msg = " Based on the available memory,"
548
- f" the estimated maximum model length is {estimated_max_len}."
547
+ estimated_msg = (
548
+ "Based on the available memory, "
549
+ f"the estimated maximum model length is {estimated_max_len}.")
550
551
raise ValueError(
552
f"To serve at least one request with the models's max seq len "
553
f"({max_model_len}), ({needed_memory/GiB_bytes:.2f} GiB KV "
554
f"cache is needed, which is larger than the available KV cache "
554
- f"memory ({available_memory/GiB_bytes:.2f} GiB)."
555
+ f"memory ({available_memory/GiB_bytes:.2f} GiB). "
556
f"{estimated_msg} "
556
- f" Try increasing `gpu_memory_utilization` or decreasing "
557
+ f"Try increasing `gpu_memory_utilization` or decreasing "
558
f"`max_model_len` when initializing the engine.")
559
560