A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 18:15 UTC
[Benchmark][Bugfix] Fix race condtion when starting server for sweep benchmark (#32927)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Jan 23, 2026 at 20:11 UTC
243e78c20fd74a68f86b6523c1f607eb3cc14ab2
2 files changed
+37
vllm/benchmarks/sweep/serve.py
+13
@@ -29,6 +29,7 @@ def run_server(
29
show_stdout: bool,
30
serve_overrides: ParameterSweepItem,
31
dry_run: bool,
32
+ server_ready_timeout: int = 300,
33
):
34
server_cmd = serve_overrides.apply_to_cmd(serve_cmd)
35
@@ -42,6 +43,7 @@ def run_server(
43
return
44
45
with ServerProcess(server_cmd, after_bench_cmd, show_stdout=show_stdout) as server:
46
+ server.wait_until_ready(timeout=server_ready_timeout)
47
yield server
48
49
print("[END SERVER]")
@@ -212,6 +214,7 @@ def run_combs(
214
num_runs: int,
215
dry_run: bool,
216
links: list[tuple[str, str]],
217
+ server_ready_timeout: int = 300,
218
):
219
all_data = list[dict[str, object]]()
220
for serve_comb in serve_params:
@@ -222,6 +225,7 @@ def run_combs(
225
show_stdout=show_stdout,
226
serve_overrides=serve_comb,
227
dry_run=dry_run,
228
+ server_ready_timeout=server_ready_timeout,
229
)
230
if _comb_needs_server(serve_comb, bench_params, output_dir)
231
else contextlib.nullcontext()
@@ -272,6 +276,7 @@ class SweepServeArgs:
276
dry_run: bool
277
resume: str | None
278
link_vars: list[tuple[str, str]] | None
279
+ server_ready_timeout: int
280
281
parser_name: ClassVar[str] = "serve"
282
parser_help: ClassVar[str] = "Run vLLM server benchmark under multiple settings."
@@ -312,6 +317,7 @@ class SweepServeArgs:
317
dry_run=args.dry_run,
318
resume=args.resume,
319
link_vars=link_vars,
320
+ server_ready_timeout=args.server_ready_timeout,
321
)
322
323
@classmethod
@@ -341,6 +347,12 @@ class SweepServeArgs:
347
help="If set, logs the standard output of subcommands. "
348
"Useful for debugging but can be quite spammy.",
349
)
350
+ parser.add_argument(
351
+ "--server-ready-timeout",
352
+ type=int,
353
+ default=300,
354
+ help="Timeout in seconds to wait for the server to become ready.",
355
+ )
356
parser.add_argument(
357
"--serve-params",
358
type=str,
@@ -431,6 +443,7 @@ def run_main(args: SweepServeArgs):
443
num_runs=args.num_runs,
444
dry_run=args.dry_run,
445
links=args.link_vars,
446
+ server_ready_timeout=args.server_ready_timeout,
447
)
448
except BaseException as exc:
449
raise RuntimeError(
vllm/benchmarks/sweep/server.py
+24
@@ -4,6 +4,7 @@ import contextlib
4
import os
5
import signal
6
import subprocess
7
+import time
8
from types import TracebackType
9
10
import requests
@@ -88,6 +89,29 @@ class ServerProcess:
89
90
return f"http://{host}:{port}"
91
92
+ def is_server_ready(self) -> bool:
93
+ server_address = self._get_vllm_server_address()
94
+ try:
95
+ response = requests.get(f"{server_address}/health")
96
+ return response.status_code == 200
97
+ except requests.RequestException:
98
+ return False
99
+
100
+ def wait_until_ready(self, timeout: int) -> None:
101
+ start_time = time.monotonic()
102
+ while not self.is_server_ready():
103
+ # Check if server process has crashed
104
+ if self._server_process.poll() is not None:
105
+ returncode = self._server_process.returncode
106
+ raise RuntimeError(
107
+ f"Server process crashed with return code {returncode}"
108
+ )
109
+ if time.monotonic() - start_time > timeout:
110
+ raise TimeoutError(
111
+ f"Server failed to become ready within {timeout} seconds."
112
+ )
113
+ time.sleep(1)
114
+
115
def reset_caches(self) -> None:
116
server_cmd = self.server_cmd
117