@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b3376e5c76
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Misc] Add args for selecting distributed executor to benchmarks (#5335)

Benjamin Kitor committed Jun 7, 2024 at 18:20 UTC b3376e5c76c199acb216addec7c32ac5299bef31
2 files changed +20 -3
benchmarks/benchmark_latency.py
+9 -1
@@ -36,7 +36,8 @@ def main(args: argparse.Namespace):
36 enable_chunked_prefill=args.enable_chunked_prefill,
37 download_dir=args.download_dir,
38 block_size=args.block_size,
39 - gpu_memory_utilization=args.gpu_memory_utilization)
39 + gpu_memory_utilization=args.gpu_memory_utilization,
40 + distributed_executor_backend=args.distributed_executor_backend)
41
42 sampling_params = SamplingParams(
43 n=args.n,
@@ -221,5 +222,12 @@ if __name__ == '__main__':
222 help='the fraction of GPU memory to be used for '
223 'the model executor, which can range from 0 to 1.'
224 'If unspecified, will use the default value of 0.9.')
225 + parser.add_argument(
226 + '--distributed-executor-backend',
227 + choices=['ray', 'mp'],
228 + default=None,
229 + help='Backend to use for distributed serving. When more than 1 GPU '
230 + 'is used, will be automatically set to "ray" if installed '
231 + 'or "mp" (multiprocessing) otherwise.')
232 args = parser.parse_args()
233 main(args)
benchmarks/benchmark_throughput.py
+11 -2
@@ -78,6 +78,7 @@ def run_vllm(
78 enable_prefix_caching: bool,
79 enable_chunked_prefill: bool,
80 max_num_batched_tokens: int,
81 + distributed_executor_backend: Optional[str],
82 gpu_memory_utilization: float = 0.9,
83 download_dir: Optional[str] = None,
84 ) -> float:
@@ -100,6 +101,7 @@ def run_vllm(
101 download_dir=download_dir,
102 enable_chunked_prefill=enable_chunked_prefill,
103 max_num_batched_tokens=max_num_batched_tokens,
104 + distributed_executor_backend=distributed_executor_backend,
105 )
106
107 # Add the requests to the engine.
@@ -225,8 +227,8 @@ def main(args: argparse.Namespace):
227 args.enforce_eager, args.kv_cache_dtype,
228 args.quantization_param_path, args.device,
229 args.enable_prefix_caching, args.enable_chunked_prefill,
228 - args.max_num_batched_tokens, args.gpu_memory_utilization,
229 - args.download_dir)
230 + args.max_num_batched_tokens, args.distributed_executor_backend,
231 + args.gpu_memory_utilization, args.download_dir)
232 elif args.backend == "hf":
233 assert args.tensor_parallel_size == 1
234 elapsed_time = run_hf(requests, args.model, tokenizer, args.n,
@@ -368,6 +370,13 @@ if __name__ == "__main__":
370 type=str,
371 default=None,
372 help='Path to save the throughput results in JSON format.')
373 + parser.add_argument(
374 + '--distributed-executor-backend',
375 + choices=['ray', 'mp'],
376 + default=None,
377 + help='Backend to use for distributed serving. When more than 1 GPU '
378 + 'is used, will be automatically set to "ray" if installed '
379 + 'or "mp" (multiprocessing) otherwise.')
380 args = parser.parse_args()
381 if args.tokenizer is None:
382 args.tokenizer = args.model