A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Update Help Text for --gpu-memory-utilization Argument (#2183)
Suhong Moon committed
Dec 18, 2023 at 14:33 UTC
290e015c6c328d08d6074827c5c43371017b27ab
1 file changed
+7
-5
vllm/engine/arg_utils.py
+7
-5
@@ -156,11 +156,13 @@ class EngineArgs:
156
type=int,
157
default=EngineArgs.swap_space,
158
help='CPU swap space size (GiB) per GPU')
159
- parser.add_argument('--gpu-memory-utilization',
160
- type=float,
161
- default=EngineArgs.gpu_memory_utilization,
162
- help='the percentage of GPU memory to be used for '
163
- 'the model executor')
159
+ parser.add_argument(
160
+ '--gpu-memory-utilization',
161
+ type=float,
162
+ default=EngineArgs.gpu_memory_utilization,
163
+ help='the fraction of GPU memory to be used for '
164
+ 'the model executor, which can range from 0 to 1.'
165
+ 'If unspecified, will use the default value of 0.9.')
166
parser.add_argument('--max-num-batched-tokens',
167
type=int,
168
default=EngineArgs.max_num_batched_tokens,