@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7dbe738d65
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Misc] benchmark: Add option to set max concurrency (#9390)

Signed-off-by: Russell Bryant <rbryant@redhat.com>

Russell Bryant committed Oct 18, 2024 at 14:15 UTC 7dbe738d653b563c646883c1ae6f6df927436d01
1 file changed +37 -3
benchmarks/benchmark_serving.py
+37 -3
@@ -398,6 +398,7 @@ async def benchmark(
398 selected_percentile_metrics: List[str],
399 selected_percentiles: List[str],
400 ignore_eos: bool,
401 + max_concurrency: Optional[int],
402 ):
403 if backend in ASYNC_REQUEST_FUNCS:
404 request_func = ASYNC_REQUEST_FUNCS[backend]
@@ -446,9 +447,25 @@ async def benchmark(
447 print("Profiler started")
448
449 print(f"Traffic request rate: {request_rate}")
450 + print(f"Maximum request concurrency: {max_concurrency}")
451
452 pbar = None if disable_tqdm else tqdm(total=len(input_requests))
453
454 + # This can be used once the minimum Python version is 3.10 or higher,
455 + # and it will simplify the code in limited_request_func.
456 + # semaphore = (asyncio.Semaphore(max_concurrency)
457 + # if max_concurrency else contextlib.nullcontext())
458 + semaphore = (asyncio.Semaphore(max_concurrency)
459 + if max_concurrency else None)
460 +
461 + async def limited_request_func(request_func_input, pbar):
462 + if semaphore is None:
463 + return await request_func(request_func_input=request_func_input,
464 + pbar=pbar)
465 + async with semaphore:
466 + return await request_func(request_func_input=request_func_input,
467 + pbar=pbar)
468 +
469 benchmark_start_time = time.perf_counter()
470 tasks: List[asyncio.Task] = []
471 async for request in get_request(input_requests, request_rate):
@@ -464,8 +481,8 @@ async def benchmark(
481 ignore_eos=ignore_eos)
482 tasks.append(
483 asyncio.create_task(
467 - request_func(request_func_input=request_func_input,
468 - pbar=pbar)))
484 + limited_request_func(request_func_input=request_func_input,
485 + pbar=pbar)))
486 outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)
487
488 if profile:
@@ -682,6 +699,7 @@ def main(args: argparse.Namespace):
699 float(p) for p in args.metric_percentiles.split(",")
700 ],
701 ignore_eos=args.ignore_eos,
702 + max_concurrency=args.max_concurrency,
703 ))
704
705 # Save config and results to json
@@ -711,13 +729,16 @@ def main(args: argparse.Namespace):
729 # Traffic
730 result_json["request_rate"] = (
731 args.request_rate if args.request_rate < float("inf") else "inf")
732 + result_json["max_concurrency"] = args.max_concurrency
733
734 # Merge with benchmark result
735 result_json = {**result_json, **benchmark_result}
736
737 # Save to file
738 base_model_id = model_id.split("/")[-1]
720 - file_name = f"{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json" #noqa
739 + max_concurrency_str = (f"-concurrency{args.max_concurrency}"
740 + if args.max_concurrency is not None else "")
741 + file_name = f"{backend}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" #noqa
742 if args.result_filename:
743 file_name = args.result_filename
744 if args.result_dir:
@@ -768,6 +789,19 @@ if __name__ == "__main__":
789 default=None,
790 help="Path to the sharegpt/sonnet dataset. "
791 "Or the huggingface dataset ID if using HF dataset.")
792 + parser.add_argument(
793 + "--max-concurrency",
794 + type=int,
795 + default=None,
796 + help="Maximum number of concurrent requests. This can be used "
797 + "to help simulate an environment where a higher level component "
798 + "is enforcing a maximum number of concurrent requests. While the "
799 + "--request-rate argument controls the rate at which requests are "
800 + "initiated, this argument will control how many are actually allowed "
801 + "to execute at a time. This means that when used in combination, the "
802 + "actual request rate may be lower than specified with --request-rate, "
803 + "if the server is not processing requests fast enough to keep up.")
804 +
805 parser.add_argument(
806 "--model",
807 type=str,