A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc] add gpu_memory_utilization arg (#5079)
Signed-off-by: pandyamarut <pandyamarut@gmail.com>
Marut Pandya committed
May 28, 2024 at 17:16 UTC
616e600e0b092050213e79fd2a10baabb30dcf6d
1 file changed
+8
-1
benchmarks/benchmark_latency.py
+8
-1
@@ -35,7 +35,8 @@ def main(args: argparse.Namespace):
35
use_v2_block_manager=args.use_v2_block_manager,
36
enable_chunked_prefill=args.enable_chunked_prefill,
37
download_dir=args.download_dir,
38
- block_size=args.block_size)
38
+ block_size=args.block_size,
39
+ gpu_memory_utilization=args.gpu_memory_utilization)
40
41
sampling_params = SamplingParams(
42
n=args.n,
@@ -214,5 +215,11 @@ if __name__ == '__main__':
215
type=str,
216
default=None,
217
help='Path to save the latency results in JSON format.')
218
+ parser.add_argument('--gpu-memory-utilization',
219
+ type=float,
220
+ default=0.9,
221
+ help='the fraction of GPU memory to be used for '
222
+ 'the model executor, which can range from 0 to 1.'
223
+ 'If unspecified, will use the default value of 0.9.')
224
args = parser.parse_args()
225
main(args)