A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Misc] Add args for selecting distributed executor to benchmarks (#5335)
Benjamin Kitor committed
Jun 7, 2024 at 18:20 UTC
b3376e5c76c199acb216addec7c32ac5299bef31
2 files changed
+20
-3
benchmarks/benchmark_latency.py
+9
-1
@@ -36,7 +36,8 @@ def main(args: argparse.Namespace):
36
enable_chunked_prefill=args.enable_chunked_prefill,
37
download_dir=args.download_dir,
38
block_size=args.block_size,
39
- gpu_memory_utilization=args.gpu_memory_utilization)
39
+ gpu_memory_utilization=args.gpu_memory_utilization,
40
+ distributed_executor_backend=args.distributed_executor_backend)
41
42
sampling_params = SamplingParams(
43
n=args.n,
@@ -221,5 +222,12 @@ if __name__ == '__main__':
222
help='the fraction of GPU memory to be used for '
223
'the model executor, which can range from 0 to 1.'
224
'If unspecified, will use the default value of 0.9.')
225
+ parser.add_argument(
226
+ '--distributed-executor-backend',
227
+ choices=['ray', 'mp'],
228
+ default=None,
229
+ help='Backend to use for distributed serving. When more than 1 GPU '
230
+ 'is used, will be automatically set to "ray" if installed '
231
+ 'or "mp" (multiprocessing) otherwise.')
232
args = parser.parse_args()
233
main(args)
benchmarks/benchmark_throughput.py
+11
-2
@@ -78,6 +78,7 @@ def run_vllm(
78
enable_prefix_caching: bool,
79
enable_chunked_prefill: bool,
80
max_num_batched_tokens: int,
81
+ distributed_executor_backend: Optional[str],
82
gpu_memory_utilization: float = 0.9,
83
download_dir: Optional[str] = None,
84
) -> float:
@@ -100,6 +101,7 @@ def run_vllm(
101
download_dir=download_dir,
102
enable_chunked_prefill=enable_chunked_prefill,
103
max_num_batched_tokens=max_num_batched_tokens,
104
+ distributed_executor_backend=distributed_executor_backend,
105
)
106
107
# Add the requests to the engine.
@@ -225,8 +227,8 @@ def main(args: argparse.Namespace):
227
args.enforce_eager, args.kv_cache_dtype,
228
args.quantization_param_path, args.device,
229
args.enable_prefix_caching, args.enable_chunked_prefill,
228
- args.max_num_batched_tokens, args.gpu_memory_utilization,
229
- args.download_dir)
230
+ args.max_num_batched_tokens, args.distributed_executor_backend,
231
+ args.gpu_memory_utilization, args.download_dir)
232
elif args.backend == "hf":
233
assert args.tensor_parallel_size == 1
234
elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -368,6 +370,13 @@ if __name__ == "__main__":
370
type=str,
371
default=None,
372
help='Path to save the throughput results in JSON format.')
373
+ parser.add_argument(
374
+ '--distributed-executor-backend',
375
+ choices=['ray', 'mp'],
376
+ default=None,
377
+ help='Backend to use for distributed serving. When more than 1 GPU '
378
+ 'is used, will be automatically set to "ray" if installed '
379
+ 'or "mp" (multiprocessing) otherwise.')
380
args = parser.parse_args()
381
if args.tokenizer is None:
382
args.tokenizer = args.model