A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Misc] benchmark: Add option to set max concurrency (#9390)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Oct 18, 2024 at 14:15 UTC
7dbe738d653b563c646883c1ae6f6df927436d01
1 file changed
+37
-3
benchmarks/benchmark_serving.py
+37
-3
@@ -398,6 +398,7 @@ async def benchmark(
398
selected_percentile_metrics: List[str],
399
selected_percentiles: List[str],
400
ignore_eos: bool,
401
+ max_concurrency: Optional[int],
402
):
403
if backend in ASYNC_REQUEST_FUNCS:
404
request_func = ASYNC_REQUEST_FUNCS[backend]
@@ -446,9 +447,25 @@ async def benchmark(
447
print("Profiler started")
448
449
print(f"Traffic request rate: {request_rate}")
450
+ print(f"Maximum request concurrency: {max_concurrency}")
451
452
pbar = None if disable_tqdm else tqdm(total=len(input_requests))
453
454
+ # This can be used once the minimum Python version is 3.10 or higher,
455
+ # and it will simplify the code in limited_request_func.
456
+ # semaphore = (asyncio.Semaphore(max_concurrency)
457
+ # if max_concurrency else contextlib.nullcontext())
458
+ semaphore = (asyncio.Semaphore(max_concurrency)
459
+ if max_concurrency else None)
460
+
461
+ async def limited_request_func(request_func_input, pbar):
462
+ if semaphore is None:
463
+ return await request_func(request_func_input=request_func_input,
464
+ pbar=pbar)
465
+ async with semaphore:
466
+ return await request_func(request_func_input=request_func_input,
467
+ pbar=pbar)
468
+
469
benchmark_start_time = time.perf_counter()
470
tasks: List[asyncio.Task] = []
471
async for request in get_request(input_requests, request_rate):
@@ -464,8 +481,8 @@ async def benchmark(
481
ignore_eos=ignore_eos)
482
tasks.append(
483
asyncio.create_task(
467
- request_func(request_func_input=request_func_input,
468
- pbar=pbar)))
484
+ limited_request_func(request_func_input=request_func_input,
485
+ pbar=pbar)))
486
outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)
487
488
if profile:
@@ -682,6 +699,7 @@ def main(args: argparse.Namespace):
699
float(p) for p in args.metric_percentiles.split(",")
700
],
701
ignore_eos=args.ignore_eos,
702
+ max_concurrency=args.max_concurrency,
703
))
704
705
# Save config and results to json
@@ -711,13 +729,16 @@ def main(args: argparse.Namespace):
729
# Traffic
730
result_json["request_rate"] = (
731
args.request_rate if args.request_rate < float("inf") else "inf")
732
+ result_json["max_concurrency"] = args.max_concurrency
733
734
# Merge with benchmark result
735
result_json = {**result_json, **benchmark_result}
736
737
# Save to file
738
base_model_id = model_id.split("/")[-1]
720
- file_name = f"{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json" #noqa
739
+ max_concurrency_str = (f"-concurrency{args.max_concurrency}"
740
+ if args.max_concurrency is not None else "")
741
+ file_name = f"{backend}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" #noqa
742
if args.result_filename:
743
file_name = args.result_filename
744
if args.result_dir:
@@ -768,6 +789,19 @@ if __name__ == "__main__":
789
default=None,
790
help="Path to the sharegpt/sonnet dataset. "
791
"Or the huggingface dataset ID if using HF dataset.")
792
+ parser.add_argument(
793
+ "--max-concurrency",
794
+ type=int,
795
+ default=None,
796
+ help="Maximum number of concurrent requests. This can be used "
797
+ "to help simulate an environment where a higher level component "
798
+ "is enforcing a maximum number of concurrent requests. While the "
799
+ "--request-rate argument controls the rate at which requests are "
800
+ "initiated, this argument will control how many are actually allowed "
801
+ "to execute at a time. This means that when used in combination, the "
802
+ "actual request rate may be lower than specified with --request-rate, "
803
+ "if the server is not processing requests fast enough to keep up.")
804
+
805
parser.add_argument(
806
"--model",
807
type=str,