A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Reduce GPU memory utilization to make sure OOM doesn't happen (#153)
Zhuohan Li committed
Jun 18, 2023 at 17:33 UTC
bf5f121c0284a2a06483b585f0d49e8508c69573
1 file changed
+1
-1
vllm/engine/arg_utils.py
+1
-1
@@ -21,7 +21,7 @@ class EngineArgs:
21
tensor_parallel_size: int = 1
22
block_size: int = 16
23
swap_space: int = 4 # GiB
24
- gpu_memory_utilization: float = 0.95
24
+ gpu_memory_utilization: float = 0.90
25
max_num_batched_tokens: int = 2560
26
max_num_seqs: int = 256
27
disable_log_stats: bool = False