A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add warning on CUDA graph memory usage (#2182)
Woosuk Kwon committed
Dec 18, 2023 at 18:16 UTC
21d5daa4aca6e16c0c42dbfdf704fdfd0006ba4c
1 file changed
+3
vllm/worker/model_runner.py
+3
@@ -395,6 +395,9 @@ class ModelRunner:
395
"unexpected consequences if the model is not static. To "
396
"run the model in eager mode, set 'enforce_eager=True' or "
397
"use '--enforce-eager' in the CLI.")
398
+ logger.info("CUDA graphs can take additional 1~3 GiB memory per GPU. "
399
+ "If you are running out of memory, consider decreasing "
400
+ "`gpu_memory_utilization` or enforcing eager mode.")
401
start_time = time.perf_counter()
402
403
# Prepare dummy inputs. These will be reused for all batch sizes.