@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8041b7305e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[BugFix] Raise error when max_model_len is larger than KV cache (#2163)

Woosuk Kwon committed Dec 17, 2023 at 17:08 UTC 8041b7305e93a8626d85cb23b3fcb995882867c1
1 file changed +8
vllm/engine/llm_engine.py
+8
@@ -227,6 +227,14 @@ class LLMEngine:
227 raise ValueError("No available memory for the cache blocks. "
228 "Try increasing `gpu_memory_utilization` when "
229 "initializing the engine.")
230 + max_seq_len = self.cache_config.block_size * num_gpu_blocks
231 + if self.model_config.max_model_len > max_seq_len:
232 + raise ValueError(
233 + f"The model's max seq len ({self.model_config.max_model_len}) "
234 + "is larger than the maximum number of tokens that can be "
235 + f"stored in KV cache ({max_seq_len}). Try increasing "
236 + "`gpu_memory_utilization` or decreasing `max_model_len` when "
237 + "initializing the engine.")
238
239 self.cache_config.num_gpu_blocks = num_gpu_blocks
240 self.cache_config.num_cpu_blocks = num_cpu_blocks