@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3a4e10c847
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 10, 04:00 UTC

[Benchmark] [Feature] add vllm bench sweep startup command (#32337)

Signed-off-by: lengrongfu <lenronfu@gmail.com>

rongfu.leng committed Jan 15, 2026 at 17:25 UTC 3a4e10c8477c329b9e75ba55ff205a1f258cbd01
3 files changed +465
docs/benchmarking/sweeps.md
+57
@@ -139,6 +139,63 @@ The algorithm for adjusting the SLA variable is as follows:
139
140 For a given combination of `--serve-params` and `--bench-params`, we share the benchmark results across `--sla-params` to avoid rerunning benchmarks with the same SLA variable value.
141
142 +### Startup
143 +
144 +`vllm bench sweep startup` runs `vllm bench startup` across parameter combinations to compare cold/warm startup time for different engine settings.
145 +
146 +Follow these steps to run the script:
147 +
148 +1. (Optional) Construct the base command to `vllm bench startup`, and pass it to `--startup-cmd` (default: `vllm bench startup`).
149 +2. (Optional) Reuse a `--serve-params` JSON from `vllm bench sweep serve` to vary engine settings. Only parameters supported by `vllm bench startup` are applied.
150 +3. (Optional) Create a `--startup-params` JSON to vary startup-specific options like iteration counts.
151 +4. Determine where you want to save the results, and pass that to `--output-dir`.
152 +
153 +Example `--serve-params`:
154 +
155 +```json
156 +[
157 + {
158 + "_benchmark_name": "tp1",
159 + "model": "Qwen/Qwen3-0.6B",
160 + "tensor_parallel_size": 1,
161 + "gpu_memory_utilization": 0.9
162 + },
163 + {
164 + "_benchmark_name": "tp2",
165 + "model": "Qwen/Qwen3-0.6B",
166 + "tensor_parallel_size": 2,
167 + "gpu_memory_utilization": 0.9
168 + }
169 +]
170 +```
171 +
172 +Example `--startup-params`:
173 +
174 +```json
175 +[
176 + {
177 + "_benchmark_name": "qwen3-0.6",
178 + "num_iters_cold": 2,
179 + "num_iters_warmup": 1,
180 + "num_iters_warm": 2
181 + }
182 +]
183 +```
184 +
185 +Example command:
186 +
187 +```bash
188 +vllm bench sweep startup \
189 + --startup-cmd 'vllm bench startup --model Qwen/Qwen3-0.6B' \
190 + --serve-params benchmarks/serve_hparams.json \
191 + --startup-params benchmarks/startup_hparams.json \
192 + -o benchmarks/results
193 +```
194 +
195 +!!! important
196 + By default, unsupported parameters in `--serve-params` or `--startup-params` are ignored with a warning.
197 + Use `--strict-params` to fail fast on unknown keys.
198 +
199 ## Visualization
200
201 ### Basic
vllm/benchmarks/sweep/cli.py
+3
@@ -12,10 +12,13 @@ from .serve import SweepServeArgs
12 from .serve import main as serve_main
13 from .serve_sla import SweepServeSLAArgs
14 from .serve_sla import main as serve_sla_main
15 +from .startup import SweepStartupArgs
16 +from .startup import main as startup_main
17
18 SUBCOMMANDS = (
19 (SweepServeArgs, serve_main),
20 (SweepServeSLAArgs, serve_sla_main),
21 + (SweepStartupArgs, startup_main),
22 (SweepPlotArgs, plot_main),
23 (SweepPlotParetoArgs, plot_pareto_main),
24 )
vllm/benchmarks/sweep/startup.py new
+405
@@ -0,0 +1,405 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import argparse
4 +import json
5 +import shlex
6 +import subprocess
7 +from dataclasses import dataclass
8 +from datetime import datetime
9 +from functools import lru_cache
10 +from pathlib import Path
11 +from typing import ClassVar
12 +
13 +from vllm.benchmarks.startup import add_cli_args as add_startup_cli_args
14 +from vllm.utils.argparse_utils import FlexibleArgumentParser
15 +from vllm.utils.import_utils import PlaceholderModule
16 +
17 +from .param_sweep import ParameterSweep, ParameterSweepItem
18 +from .utils import sanitize_filename
19 +
20 +try:
21 + import pandas as pd
22 +except ImportError:
23 + pd = PlaceholderModule("pandas")
24 +
25 +
26 +@lru_cache(maxsize=1)
27 +def _get_supported_startup_keys() -> set[str]:
28 + parser = FlexibleArgumentParser(add_help=False)
29 + add_startup_cli_args(parser)
30 +
31 + supported: set[str] = {"config"}
32 + for action in parser._actions:
33 + if action.dest and action.dest is not argparse.SUPPRESS:
34 + supported.add(action.dest)
35 + for option in action.option_strings:
36 + if option.startswith("--"):
37 + supported.add(option.lstrip("-").replace("-", "_"))
38 +
39 + return supported
40 +
41 +
42 +def _is_supported_param(param_key: str, supported: set[str]) -> bool:
43 + if param_key == "_benchmark_name":
44 + return True
45 + prefix = param_key.split(".", 1)[0]
46 + normalized = prefix.replace("-", "_")
47 + return normalized in supported
48 +
49 +
50 +def _filter_params(
51 + params: ParameterSweep, *, supported: set[str], strict: bool
52 +) -> ParameterSweep:
53 + filtered = []
54 + for item in params:
55 + kept: dict[str, object] = {}
56 + dropped: list[str] = []
57 + for key, value in item.items():
58 + if _is_supported_param(key, supported):
59 + kept[key] = value
60 + else:
61 + dropped.append(key)
62 +
63 + if dropped:
64 + label = item.get("_benchmark_name") or item.as_text()
65 + message = (
66 + "Ignoring unsupported startup params"
67 + f"{' for ' + str(label) if label else ''}: "
68 + f"{', '.join(sorted(dropped))}"
69 + )
70 + if strict:
71 + raise ValueError(message)
72 + print(message)
73 +
74 + filtered.append(ParameterSweepItem.from_record(kept))
75 +
76 + return ParameterSweep(filtered)
77 +
78 +
79 +def _update_run_data(
80 + run_data: dict[str, object],
81 + serve_overrides: ParameterSweepItem,
82 + startup_overrides: ParameterSweepItem,
83 + run_number: int,
84 +) -> dict[str, object]:
85 + run_data["run_number"] = run_number
86 + run_data.update(serve_overrides)
87 + run_data.update(startup_overrides)
88 + return run_data
89 +
90 +
91 +def _strip_arg(cmd: list[str], keys: tuple[str, ...]) -> list[str]:
92 + stripped: list[str] = []
93 + skip_next = False
94 + for arg in cmd:
95 + if skip_next:
96 + skip_next = False
97 + continue
98 + if arg in keys:
99 + skip_next = True
100 + continue
101 + if any(arg.startswith(f"{key}=") for key in keys):
102 + continue
103 + stripped.append(arg)
104 + return stripped
105 +
106 +
107 +def _apply_output_json(cmd: list[str], output_path: Path) -> list[str]:
108 + keys = ("--output-json", "--output_json")
109 + cmd = _strip_arg(cmd, keys)
110 + return [*cmd, keys[0], str(output_path)]
111 +
112 +
113 +def _get_comb_base_path(
114 + output_dir: Path,
115 + serve_comb: ParameterSweepItem,
116 + startup_comb: ParameterSweepItem,
117 +) -> Path:
118 + parts = list[str]()
119 + if serve_comb:
120 + parts.extend(("SERVE-", serve_comb.name))
121 + if startup_comb:
122 + parts.extend(("STARTUP-", startup_comb.name))
123 + return output_dir / sanitize_filename("-".join(parts))
124 +
125 +
126 +def _get_comb_run_path(base_path: Path, run_number: int | None) -> Path:
127 + if run_number is None:
128 + return base_path / "summary.json"
129 + return base_path / f"run={run_number}.json"
130 +
131 +
132 +def run_benchmark(
133 + startup_cmd: list[str],
134 + *,
135 + serve_overrides: ParameterSweepItem,
136 + startup_overrides: ParameterSweepItem,
137 + run_number: int,
138 + output_path: Path,
139 + show_stdout: bool,
140 + dry_run: bool,
141 +) -> dict[str, object] | None:
142 + cmd = serve_overrides.apply_to_cmd(startup_cmd)
143 + cmd = startup_overrides.apply_to_cmd(cmd)
144 + cmd = _apply_output_json(cmd, output_path)
145 +
146 + print("[BEGIN BENCHMARK]")
147 + print(f"Serve overrides: {serve_overrides}")
148 + print(f"Startup overrides: {startup_overrides}")
149 + print(f"Run Number: {run_number}")
150 + print(f"Benchmark command: {cmd}")
151 + print(f"Output file: {output_path}")
152 +
153 + if output_path.exists():
154 + print("Found existing results. Skipping.")
155 +
156 + with output_path.open("r", encoding="utf-8") as f:
157 + run_data = json.load(f)
158 + return _update_run_data(
159 + run_data, serve_overrides, startup_overrides, run_number
160 + )
161 +
162 + if dry_run:
163 + print("[END BENCHMARK]")
164 + return None
165 +
166 + output_path.parent.mkdir(parents=True, exist_ok=True)
167 + subprocess.run(
168 + cmd,
169 + stdout=None if show_stdout else subprocess.DEVNULL,
170 + check=True,
171 + )
172 +
173 + with output_path.open("r", encoding="utf-8") as f:
174 + run_data = json.load(f)
175 +
176 + run_data = _update_run_data(
177 + run_data, serve_overrides, startup_overrides, run_number
178 + )
179 +
180 + with output_path.open("w", encoding="utf-8") as f:
181 + json.dump(run_data, f, indent=4)
182 +
183 + print("[END BENCHMARK]")
184 + return run_data
185 +
186 +
187 +def run_comb(
188 + startup_cmd: list[str],
189 + *,
190 + serve_comb: ParameterSweepItem,
191 + startup_comb: ParameterSweepItem,
192 + base_path: Path,
193 + num_runs: int,
194 + show_stdout: bool,
195 + dry_run: bool,
196 +) -> list[dict[str, object]] | None:
197 + comb_data = list[dict[str, object]]()
198 + for run_number in range(num_runs):
199 + run_data = run_benchmark(
200 + startup_cmd,
201 + serve_overrides=serve_comb,
202 + startup_overrides=startup_comb,
203 + run_number=run_number,
204 + output_path=_get_comb_run_path(base_path, run_number),
205 + show_stdout=show_stdout,
206 + dry_run=dry_run,
207 + )
208 + if run_data is not None:
209 + comb_data.append(run_data)
210 +
211 + if dry_run:
212 + return None
213 +
214 + with _get_comb_run_path(base_path, run_number=None).open(
215 + "w", encoding="utf-8"
216 + ) as f:
217 + json.dump(comb_data, f, indent=4)
218 +
219 + return comb_data
220 +
221 +
222 +def run_combs(
223 + startup_cmd: list[str],
224 + *,
225 + serve_params: ParameterSweep,
226 + startup_params: ParameterSweep,
227 + output_dir: Path,
228 + num_runs: int,
229 + show_stdout: bool,
230 + dry_run: bool,
231 +) -> "pd.DataFrame | None":
232 + all_data = list[dict[str, object]]()
233 + for serve_comb in serve_params:
234 + for startup_comb in startup_params:
235 + base_path = _get_comb_base_path(output_dir, serve_comb, startup_comb)
236 + comb_data = run_comb(
237 + startup_cmd,
238 + serve_comb=serve_comb,
239 + startup_comb=startup_comb,
240 + base_path=base_path,
241 + num_runs=num_runs,
242 + show_stdout=show_stdout,
243 + dry_run=dry_run,
244 + )
245 + if comb_data is not None:
246 + all_data.extend(comb_data)
247 +
248 + if dry_run:
249 + return None
250 +
251 + combined_df = pd.DataFrame.from_records(all_data)
252 + combined_df.to_csv(output_dir / "summary.csv")
253 + return combined_df
254 +
255 +
256 +@dataclass
257 +class SweepStartupArgs:
258 + startup_cmd: list[str]
259 + serve_params: ParameterSweep
260 + startup_params: ParameterSweep
261 + output_dir: Path
262 + num_runs: int
263 + show_stdout: bool
264 + dry_run: bool
265 + resume: str | None
266 + strict_params: bool
267 +
268 + parser_name: ClassVar[str] = "startup"
269 + parser_help: ClassVar[str] = (
270 + "Benchmark vLLM startup time over parameter combinations."
271 + )
272 +
273 + @classmethod
274 + def from_cli_args(cls, args: argparse.Namespace):
275 + startup_cmd = shlex.split(args.startup_cmd)
276 +
277 + if args.serve_params:
278 + serve_params = ParameterSweep.read_json(args.serve_params)
279 + else:
280 + serve_params = ParameterSweep.from_records([{}])
281 +
282 + if args.startup_params:
283 + startup_params = ParameterSweep.read_json(args.startup_params)
284 + else:
285 + startup_params = ParameterSweep.from_records([{}])
286 +
287 + supported = _get_supported_startup_keys()
288 + serve_params = _filter_params(
289 + serve_params, supported=supported, strict=args.strict_params
290 + )
291 + startup_params = _filter_params(
292 + startup_params, supported=supported, strict=args.strict_params
293 + )
294 +
295 + if args.num_runs < 1:
296 + raise ValueError("`num_runs` should be at least 1.")
297 +
298 + return cls(
299 + startup_cmd=startup_cmd,
300 + serve_params=serve_params,
301 + startup_params=startup_params,
302 + output_dir=Path(args.output_dir),
303 + num_runs=args.num_runs,
304 + show_stdout=args.show_stdout,
305 + dry_run=args.dry_run,
306 + resume=args.resume,
307 + strict_params=args.strict_params,
308 + )
309 +
310 + @classmethod
311 + def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
312 + parser.add_argument(
313 + "--startup-cmd",
314 + type=str,
315 + default="vllm bench startup",
316 + help="The command used to run the startup benchmark.",
317 + )
318 + parser.add_argument(
319 + "--serve-params",
320 + type=str,
321 + default=None,
322 + help="Path to JSON file containing parameter combinations "
323 + "for the `vllm serve` command. Only parameters supported by "
324 + "`vllm bench startup` will be applied.",
325 + )
326 + parser.add_argument(
327 + "--startup-params",
328 + type=str,
329 + default=None,
330 + help="Path to JSON file containing parameter combinations "
331 + "for the `vllm bench startup` command.",
332 + )
333 + parser.add_argument(
334 + "-o",
335 + "--output-dir",
336 + type=str,
337 + default="results",
338 + help="The directory to which results are written.",
339 + )
340 + parser.add_argument(
341 + "--num-runs",
342 + type=int,
343 + default=1,
344 + help="Number of runs per parameter combination.",
345 + )
346 + parser.add_argument(
347 + "--show-stdout",
348 + action="store_true",
349 + help="If set, logs the standard output of subcommands.",
350 + )
351 + parser.add_argument(
352 + "--dry-run",
353 + action="store_true",
354 + help="If set, prints the commands to run, "
355 + "then exits without executing them.",
356 + )
357 + parser.add_argument(
358 + "--resume",
359 + type=str,
360 + default=None,
361 + help="Set this to the name of a directory under `output_dir` (which is a "
362 + "timestamp) to resume a previous execution of this script, i.e., only run "
363 + "parameter combinations for which there are still no output files.",
364 + )
365 + parser.add_argument(
366 + "--strict-params",
367 + action="store_true",
368 + help="If set, unknown parameters in sweep files raise an error "
369 + "instead of being ignored.",
370 + )
371 + return parser
372 +
373 +
374 +def run_main(args: SweepStartupArgs):
375 + timestamp = args.resume or datetime.now().strftime("%Y%m%d_%H%M%S")
376 + output_dir = args.output_dir / timestamp
377 +
378 + if args.resume and not output_dir.exists():
379 + raise ValueError(f"Cannot resume from non-existent directory ({output_dir})")
380 +
381 + try:
382 + return run_combs(
383 + startup_cmd=args.startup_cmd,
384 + serve_params=args.serve_params,
385 + startup_params=args.startup_params,
386 + output_dir=output_dir,
387 + num_runs=args.num_runs,
388 + show_stdout=args.show_stdout,
389 + dry_run=args.dry_run,
390 + )
391 + except BaseException as exc:
392 + raise RuntimeError(
393 + f"The script was terminated early. Use `--resume {timestamp}` "
394 + f"to continue the script from its last checkpoint."
395 + ) from exc
396 +
397 +
398 +def main(args: argparse.Namespace):
399 + run_main(SweepStartupArgs.from_cli_args(args))
400 +
401 +
402 +if __name__ == "__main__":
403 + parser = argparse.ArgumentParser(description=SweepStartupArgs.parser_help)
404 + SweepStartupArgs.add_cli_args(parser)
405 + main(parser.parse_args())