A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[Bug] Fix the OOM condition for CPU cache (#260)
Zhuohan Li committed
Jun 26, 2023 at 11:16 UTC
0b7db411b5af4bf8a3a0cf989daa024ba5401ac1
2 files changed
+3
-1
vllm/engine/llm_engine.py
+1
-1
@@ -128,7 +128,7 @@ class LLMEngine:
128
logger.info(f'# GPU blocks: {num_gpu_blocks}, '
129
f'# CPU blocks: {num_cpu_blocks}')
130
131
- if num_gpu_blocks <= 0 or num_cpu_blocks <= 0:
131
+ if num_gpu_blocks <= 0:
132
raise ValueError("No available memory for the cache blocks. "
133
"Try increasing `gpu_memory_utilization` when "
134
"initializing the engine.")
vllm/worker/worker.py
+2
@@ -113,6 +113,8 @@ class Worker:
113
num_gpu_blocks = int((total_gpu_memory * gpu_memory_utilization
114
- peak_memory) // cache_block_size)
115
num_cpu_blocks = int(cpu_swap_space // cache_block_size)
116
+ num_gpu_blocks = max(num_gpu_blocks, 0)
117
+ num_cpu_blocks = max(num_cpu_blocks, 0)
118
torch.cuda.empty_cache()
119
120
# Reset the seed to ensure that the random state is not affected by