A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
enable --gpu-memory-utilization in benchmark_throughput.py (#3175)
Co-authored-by: zixiao <shunli.dsl@alibaba-inc.com>
Allen.Dou committed
Mar 5, 2024 at 02:37 UTC
9cbc7e5f3be72552d6041f81738921a9597643e8
1 file changed
+14
-7
benchmarks/benchmark_throughput.py
+14
-7
@@ -74,6 +74,7 @@ def run_vllm(
74
kv_cache_dtype: str,
75
device: str,
76
enable_prefix_caching: bool,
77
+ gpu_memory_utilization: float = 0.9,
78
) -> float:
79
from vllm import LLM, SamplingParams
80
llm = LLM(model=model,
@@ -84,6 +85,7 @@ def run_vllm(
85
trust_remote_code=trust_remote_code,
86
dtype=dtype,
87
max_model_len=max_model_len,
88
+ gpu_memory_utilization=gpu_memory_utilization,
89
enforce_eager=enforce_eager,
90
kv_cache_dtype=kv_cache_dtype,
91
device=device,
@@ -206,13 +208,12 @@ def main(args: argparse.Namespace):
208
args.output_len)
209
210
if args.backend == "vllm":
209
- elapsed_time = run_vllm(requests, args.model, args.tokenizer,
210
- args.quantization, args.tensor_parallel_size,
211
- args.seed, args.n, args.use_beam_search,
212
- args.trust_remote_code, args.dtype,
213
- args.max_model_len, args.enforce_eager,
214
- args.kv_cache_dtype, args.device,
215
- args.enable_prefix_caching)
211
+ elapsed_time = run_vllm(
212
+ requests, args.model, args.tokenizer, args.quantization,
213
+ args.tensor_parallel_size, args.seed, args.n, args.use_beam_search,
214
+ args.trust_remote_code, args.dtype, args.max_model_len,
215
+ args.enforce_eager, args.kv_cache_dtype, args.device,
216
+ args.enable_prefix_caching, args.gpu_memory_utilization)
217
elif args.backend == "hf":
218
assert args.tensor_parallel_size == 1
219
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -287,6 +288,12 @@ if __name__ == "__main__":
288
'The "auto" option will use FP16 precision '
289
'for FP32 and FP16 models, and BF16 precision '
290
'for BF16 models.')
291
+ parser.add_argument('--gpu-memory-utilization',
292
+ type=float,
293
+ default=0.9,
294
+ help='the fraction of GPU memory to be used for '
295
+ 'the model executor, which can range from 0 to 1.'
296
+ 'If unspecified, will use the default value of 0.9.')
297
parser.add_argument("--enforce-eager",
298
action="store_true",
299
help="enforce eager execution")