A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Adding Warmup to Benchmark Serving (#26943)
Signed-off-by: Kimbo Chen <chentenghung@gmail.com>
kimbochen committed
Oct 16, 2025 at 15:44 UTC
013abde6ef02e55abb393d3baff855eea9693479
1 file changed
+36
-1
vllm/benchmarks/serve.py
+36
-1
@@ -478,6 +478,7 @@ async def benchmark(
478
request_rate: float,
479
burstiness: float,
480
disable_tqdm: bool,
481
+ num_warmups: int,
482
profile: bool,
483
selected_percentile_metrics: list[str],
484
selected_percentiles: list[float],
@@ -559,10 +560,37 @@ async def benchmark(
560
f"Error: {test_output.error}"
561
)
562
else:
562
- print("Initial test run completed. Starting main benchmark run...")
563
+ print("Initial test run completed.")
564
else:
565
print("Skipping endpoint ready check.")
566
567
+ if num_warmups > 0:
568
+ print(f"Warming up with {num_warmups} requests...")
569
+ warmup_pbar = None if disable_tqdm else tqdm(total=num_warmups)
570
+ warmup_semaphore = (
571
+ asyncio.Semaphore(max_concurrency)
572
+ if max_concurrency
573
+ else contextlib.nullcontext()
574
+ )
575
+ warmup_tasks = []
576
+
577
+ async def warmup_limited_request_func():
578
+ async with warmup_semaphore:
579
+ return await request_func(
580
+ request_func_input=test_input, session=session, pbar=warmup_pbar
581
+ )
582
+
583
+ for _ in range(num_warmups):
584
+ request_task = asyncio.create_task(warmup_limited_request_func())
585
+ warmup_tasks.append(request_task)
586
+ _ = await asyncio.gather(*warmup_tasks)
587
+
588
+ if warmup_pbar is not None:
589
+ warmup_pbar.close()
590
+ print("Warmup run completed.")
591
+
592
+ print("Starting main benchmark run...")
593
+
594
if lora_modules:
595
# For each input request, choose a LoRA module at random.
596
lora_modules = iter(
@@ -1029,6 +1057,12 @@ def add_cli_args(parser: argparse.ArgumentParser):
1057
action="store_true",
1058
help="Specify to disable tqdm progress bar.",
1059
)
1060
+ parser.add_argument(
1061
+ "--num-warmups",
1062
+ type=int,
1063
+ default=0,
1064
+ help="Number of warmup requests.",
1065
+ )
1066
parser.add_argument(
1067
"--profile",
1068
action="store_true",
@@ -1370,6 +1404,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1404
request_rate=args.request_rate,
1405
burstiness=args.burstiness,
1406
disable_tqdm=args.disable_tqdm,
1407
+ num_warmups=args.num_warmups,
1408
profile=args.profile,
1409
selected_percentile_metrics=percentile_metrics.split(","),
1410
selected_percentiles=[float(p) for p in args.metric_percentiles.split(",")],