A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Fix] Better error message when there is OOM during cache initialization (#203)
Zhuohan Li committed
Jun 22, 2023 at 15:30 UTC
1d24ccb96cdfbb42fc2b0f0591df82727d9537c6
2 files changed
+7
vllm/engine/llm_engine.py
+6
@@ -127,6 +127,12 @@ class LLMEngine:
127
# FIXME(woosuk): Change to debug log.
128
logger.info(f'# GPU blocks: {num_gpu_blocks}, '
129
f'# CPU blocks: {num_cpu_blocks}')
130
+
131
+ if num_gpu_blocks <= 0 or num_cpu_blocks <= 0:
132
+ raise ValueError("No available memory for the cache blocks. "
133
+ "Try increasing `gpu_memory_utilization` when "
134
+ "initializing the engine.")
135
+
136
self.cache_config.num_gpu_blocks = num_gpu_blocks
137
self.cache_config.num_cpu_blocks = num_cpu_blocks
138
vllm/outputs.py
+1
@@ -53,6 +53,7 @@ class RequestOutput:
53
prompt: The prompt string of the request.
54
prompt_token_ids: The token IDs of the prompt.
55
outputs: The output sequences of the request.
56
+ finished: Whether the whole request is finished.
57
"""
58
def __init__(
59
self,