@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 616e600e0b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc] add gpu_memory_utilization arg (#5079)

Signed-off-by: pandyamarut <pandyamarut@gmail.com>

Marut Pandya committed May 28, 2024 at 17:16 UTC 616e600e0b092050213e79fd2a10baabb30dcf6d
1 file changed +8 -1
benchmarks/benchmark_latency.py
+8 -1
@@ -35,7 +35,8 @@ def main(args: argparse.Namespace):
35 use_v2_block_manager=args.use_v2_block_manager,
36 enable_chunked_prefill=args.enable_chunked_prefill,
37 download_dir=args.download_dir,
38 - block_size=args.block_size)
38 + block_size=args.block_size,
39 + gpu_memory_utilization=args.gpu_memory_utilization)
40
41 sampling_params = SamplingParams(
42 n=args.n,
@@ -214,5 +215,11 @@ if __name__ == '__main__':
215 type=str,
216 default=None,
217 help='Path to save the latency results in JSON format.')
218 + parser.add_argument('--gpu-memory-utilization',
219 + type=float,
220 + default=0.9,
221 + help='the fraction of GPU memory to be used for '
222 + 'the model executor, which can range from 0 to 1.'
223 + 'If unspecified, will use the default value of 0.9.')
224 args = parser.parse_args()
225 main(args)