@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9cbc7e5f3b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

enable --gpu-memory-utilization in benchmark_throughput.py (#3175)

Co-authored-by: zixiao <shunli.dsl@alibaba-inc.com>

Allen.Dou committed Mar 5, 2024 at 02:37 UTC 9cbc7e5f3be72552d6041f81738921a9597643e8
1 file changed +14 -7
benchmarks/benchmark_throughput.py
+14 -7
@@ -74,6 +74,7 @@ def run_vllm(
74 kv_cache_dtype: str,
75 device: str,
76 enable_prefix_caching: bool,
77 + gpu_memory_utilization: float = 0.9,
78 ) -> float:
79 from vllm import LLM, SamplingParams
80 llm = LLM(model=model,
@@ -84,6 +85,7 @@ def run_vllm(
85 trust_remote_code=trust_remote_code,
86 dtype=dtype,
87 max_model_len=max_model_len,
88 + gpu_memory_utilization=gpu_memory_utilization,
89 enforce_eager=enforce_eager,
90 kv_cache_dtype=kv_cache_dtype,
91 device=device,
@@ -206,13 +208,12 @@ def main(args: argparse.Namespace):
208 args.output_len)
209
210 if args.backend == "vllm":
209 - elapsed_time = run_vllm(requests, args.model, args.tokenizer,
210 - args.quantization, args.tensor_parallel_size,
211 - args.seed, args.n, args.use_beam_search,
212 - args.trust_remote_code, args.dtype,
213 - args.max_model_len, args.enforce_eager,
214 - args.kv_cache_dtype, args.device,
215 - args.enable_prefix_caching)
211 + elapsed_time = run_vllm(
212 + requests, args.model, args.tokenizer, args.quantization,
213 + args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
214 + args.trust_remote_code, args.dtype, args.max_model_len,
215 + args.enforce_eager, args.kv_cache_dtype, args.device,
216 + args.enable_prefix_caching, args.gpu_memory_utilization)
217 elif args.backend == "hf":
218 assert args.tensor_parallel_size == 1
219 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -287,6 +288,12 @@ if __name__ == "__main__":
288 'The "auto" option will use FP16 precision '
289 'for FP32 and FP16 models, and BF16 precision '
290 'for BF16 models.')
291 + parser.add_argument('--gpu-memory-utilization',
292 + type=float,
293 + default=0.9,
294 + help='the fraction of GPU memory to be used for '
295 + 'the model executor, which can range from 0 to 1.'
296 + 'If unspecified, will use the default value of 0.9.')
297 parser.add_argument("--enforce-eager",
298 action="store_true",
299 help="enforce eager execution")