A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[BugFix] Raise error when max_model_len is larger than KV cache (#2163)
Woosuk Kwon committed
Dec 17, 2023 at 17:08 UTC
8041b7305e93a8626d85cb23b3fcb995882867c1
1 file changed
+8
vllm/engine/llm_engine.py
+8
@@ -227,6 +227,14 @@ class LLMEngine:
227
raise ValueError("No available memory for the cache blocks. "
228
"Try increasing `gpu_memory_utilization` when "
229
"initializing the engine.")
230
+ max_seq_len = self.cache_config.block_size * num_gpu_blocks
231
+ if self.model_config.max_model_len > max_seq_len:
232
+ raise ValueError(
233
+ f"The model's max seq len ({self.model_config.max_model_len}) "
234
+ "is larger than the maximum number of tokens that can be "
235
+ f"stored in KV cache ({max_seq_len}). Try increasing "
236
+ "`gpu_memory_utilization` or decreasing `max_model_len` when "
237
+ "initializing the engine.")
238
239
self.cache_config.num_gpu_blocks = num_gpu_blocks
240
self.cache_config.num_cpu_blocks = num_cpu_blocks