A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[benchmarks]allow skip ready check for bench serve (#25420)
Signed-off-by: Lu Fang <fanglu@fb.com> Signed-off-by: Lucia Fang <116399278+luccafong@users.noreply.github.com> Co-authored-by: Lucia (Lu) Fang <fanglu@meta.com>
Lucia Fang committed
Sep 22, 2025 at 20:21 UTC
eea178398997b94b6df5560b597e2b7895b2ae3a
1 file changed
+17
-12
vllm/benchmarks/serve.py
+17
-12
@@ -531,18 +531,22 @@ async def benchmark(
531
extra_body=extra_body,
532
)
533
534
- test_output = await wait_for_endpoint(
535
- request_func,
536
- test_input,
537
- session,
538
- timeout_seconds=ready_check_timeout_sec,
539
- )
540
- if not test_output.success:
541
- raise ValueError(
542
- "Initial test run failed - Please make sure benchmark arguments "
543
- f"are correctly specified. Error: {test_output.error}")
534
+ if ready_check_timeout_sec > 0:
535
+ test_output = await wait_for_endpoint(
536
+ request_func,
537
+ test_input,
538
+ session,
539
+ timeout_seconds=ready_check_timeout_sec,
540
+ )
541
+ if not test_output.success:
542
+ raise ValueError(
543
+ "Initial test run failed - Please make sure benchmark "
544
+ "arguments are correctly specified. "
545
+ f"Error: {test_output.error}")
546
+ else:
547
+ print("Initial test run completed. Starting main benchmark run...")
548
else:
545
- print("Initial test run completed. Starting main benchmark run...")
549
+ print("Skipping endpoint ready check.")
550
551
if lora_modules:
552
# For each input request, choose a LoRA module at random.
@@ -1151,7 +1155,8 @@ def add_cli_args(parser: argparse.ArgumentParser):
1155
type=int,
1156
default=600,
1157
help="Maximum time to wait for the endpoint to become ready "
1154
- "in seconds (default: 600 seconds / 10 minutes).",
1158
+ "in seconds (default: 600 seconds / 10 minutes). If set to 0, "
1159
+ "the ready check will be skipped."
1160
)
1161
1162