@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 82c25151ec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Doc] update gpu-memory-utilization flag docs (#9507)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Oct 18, 2024 at 22:26 UTC 82c25151ec54f723de8589ccc3ad24d4a1817e90
1 file changed +5 -1
vllm/engine/arg_utils.py
+5 -1
@@ -428,7 +428,11 @@ class EngineArgs:
428 help='The fraction of GPU memory to be used for the model '
429 'executor, which can range from 0 to 1. For example, a value of '
430 '0.5 would imply 50%% GPU memory utilization. If unspecified, '
431 - 'will use the default value of 0.9.')
431 + 'will use the default value of 0.9. This is a global gpu memory '
432 + 'utilization limit, for example if 50%% of the gpu memory is '
433 + 'already used before vLLM starts and --gpu-memory-utilization is '
434 + 'set to 0.9, then only 40%% of the gpu memory will be allocated '
435 + 'to the model executor.')
436 parser.add_argument(
437 '--num-gpu-blocks-override',
438 type=int,