@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 290e015c6c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Update Help Text for --gpu-memory-utilization Argument (#2183)

Suhong Moon committed Dec 18, 2023 at 14:33 UTC 290e015c6c328d08d6074827c5c43371017b27ab
1 file changed +7 -5
vllm/engine/arg_utils.py
+7 -5
@@ -156,11 +156,13 @@ class EngineArgs:
156 type=int,
157 default=EngineArgs.swap_space,
158 help='CPU swap space size (GiB) per GPU')
159 - parser.add_argument('--gpu-memory-utilization',
160 - type=float,
161 - default=EngineArgs.gpu_memory_utilization,
162 - help='the percentage of GPU memory to be used for '
163 - 'the model executor')
159 + parser.add_argument(
160 + '--gpu-memory-utilization',
161 + type=float,
162 + default=EngineArgs.gpu_memory_utilization,
163 + help='the fraction of GPU memory to be used for '
164 + 'the model executor, which can range from 0 to 1.'
165 + 'If unspecified, will use the default value of 0.9.')
166 parser.add_argument('--max-num-batched-tokens',
167 type=int,
168 default=EngineArgs.max_num_batched_tokens,