@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0b7db411b5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[Bug] Fix the OOM condition for CPU cache (#260)

Zhuohan Li committed Jun 26, 2023 at 11:16 UTC 0b7db411b5af4bf8a3a0cf989daa024ba5401ac1
2 files changed +3 -1
vllm/engine/llm_engine.py
+1 -1
@@ -128,7 +128,7 @@ class LLMEngine:
128 logger.info(f'# GPU blocks: {num_gpu_blocks}, '
129 f'# CPU blocks: {num_cpu_blocks}')
130
131 - if num_gpu_blocks <= 0 or num_cpu_blocks <= 0:
131 + if num_gpu_blocks <= 0:
132 raise ValueError("No available memory for the cache blocks. "
133 "Try increasing `gpu_memory_utilization` when "
134 "initializing the engine.")
vllm/worker/worker.py
+2
@@ -113,6 +113,8 @@ class Worker:
113 num_gpu_blocks = int((total_gpu_memory * gpu_memory_utilization
114 - peak_memory) // cache_block_size)
115 num_cpu_blocks = int(cpu_swap_space // cache_block_size)
116 + num_gpu_blocks = max(num_gpu_blocks, 0)
117 + num_cpu_blocks = max(num_cpu_blocks, 0)
118 torch.cuda.empty_cache()
119
120 # Reset the seed to ensure that the random state is not affected by