@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1d24ccb96c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Fix] Better error message when there is OOM during cache initialization (#203)

Zhuohan Li committed Jun 22, 2023 at 15:30 UTC 1d24ccb96cdfbb42fc2b0f0591df82727d9537c6
2 files changed +7
vllm/engine/llm_engine.py
+6
@@ -127,6 +127,12 @@ class LLMEngine:
127 # FIXME(woosuk): Change to debug log.
128 logger.info(f'# GPU blocks: {num_gpu_blocks}, '
129 f'# CPU blocks: {num_cpu_blocks}')
130 +
131 + if num_gpu_blocks <= 0 or num_cpu_blocks <= 0:
132 + raise ValueError("No available memory for the cache blocks. "
133 + "Try increasing `gpu_memory_utilization` when "
134 + "initializing the engine.")
135 +
136 self.cache_config.num_gpu_blocks = num_gpu_blocks
137 self.cache_config.num_cpu_blocks = num_cpu_blocks
138
vllm/outputs.py
+1
@@ -53,6 +53,7 @@ class RequestOutput:
53 prompt: The prompt string of the request.
54 prompt_token_ids: The token IDs of the prompt.
55 outputs: The output sequences of the request.
56 + finished: Whether the whole request is finished.
57 """
58 def __init__(
59 self,