@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 480598958e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 12, 09:45 UTC

[Feature][Bench] Add pareto visualization (#29477)

Signed-off-by: rongfu.leng <rongfu.leng@daocloud.io>

rongfu.leng committed Nov 28, 2025 at 15:53 UTC 480598958e28fa1e2ed2f7be2d457fc6f85a1748
4 files changed +418
docs/contributing/benchmarks.md
+18
@@ -1146,6 +1146,24 @@ vllm bench sweep plot benchmarks/results/<timestamp> \
1146 !!! tip
1147 You can use `--dry-run` to preview the figures to be plotted.
1148
1149 +### Pareto visualization (tokens/s/user vs tokens/s/GPU)
1150 +
1151 +`vllm bench sweep plot_pareto` helps pick configurations that balance per-user and per-GPU throughput.
1152 +
1153 +Higher concurrency or batch size can raise GPU efficiency (per-GPU), but can add per user latency; lower concurrency improves per-user rate but underutilizes GPUs; The Pareto frontier shows the best achievable pairs across your runs.
1154 +
1155 +- x-axis: tokens/s/user = `output_throughput` ÷ concurrency (`--user-count-var`, default `max_concurrency`, fallback `max_concurrent_requests`).
1156 +- y-axis: tokens/s/GPU = `output_throughput` ÷ GPU count (`--gpu-count-var` if set; else gpu_count is TP×PP*DP).
1157 +- Output: a single figure at `OUTPUT_DIR/pareto/PARETO.png`.
1158 +- Show the configuration used in each data point `--label-by` (default: `max_concurrency,gpu_count`).
1159 +
1160 +Example:
1161 +
1162 +```bash
1163 +vllm bench sweep plot_pareto benchmarks/results/<timestamp> \
1164 + --label-by max_concurrency,tensor_parallel_size,pipeline_parallel_size
1165 +```
1166 +
1167 ## Performance Benchmarks
1168
1169 The performance benchmarks are used for development to confirm whether new changes improve performance under various workloads. They are triggered on every commit with both the `perf-benchmarks` and `ready` labels, and when a PR is merged into vLLM.
docs/mkdocs/hooks/generate_argparse.py
+4
@@ -94,6 +94,9 @@ def auto_mock(module_name: str, attr: str, max_mocks: int = 100):
94 bench_latency = auto_mock("vllm.benchmarks", "latency")
95 bench_serve = auto_mock("vllm.benchmarks", "serve")
96 bench_sweep_plot = auto_mock("vllm.benchmarks.sweep.plot", "SweepPlotArgs")
97 +bench_sweep_plot_pareto = auto_mock(
98 + "vllm.benchmarks.sweep.plot_pareto", "SweepPlotParetoArgs"
99 +)
100 bench_sweep_serve = auto_mock("vllm.benchmarks.sweep.serve", "SweepServeArgs")
101 bench_sweep_serve_sla = auto_mock(
102 "vllm.benchmarks.sweep.serve_sla", "SweepServeSLAArgs"
@@ -221,6 +224,7 @@ def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
224 "bench_latency": create_parser(bench_latency.add_cli_args),
225 "bench_serve": create_parser(bench_serve.add_cli_args),
226 "bench_sweep_plot": create_parser(bench_sweep_plot.add_cli_args),
227 + "bench_sweep_plot_pareto": create_parser(bench_sweep_plot_pareto.add_cli_args),
228 "bench_sweep_serve": create_parser(bench_sweep_serve.add_cli_args),
229 "bench_sweep_serve_sla": create_parser(bench_sweep_serve_sla.add_cli_args),
230 "bench_throughput": create_parser(bench_throughput.add_cli_args),
vllm/benchmarks/sweep/cli.py
+3
@@ -6,6 +6,8 @@ from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
6
7 from .plot import SweepPlotArgs
8 from .plot import main as plot_main
9 +from .plot_pareto import SweepPlotParetoArgs
10 +from .plot_pareto import main as plot_pareto_main
11 from .serve import SweepServeArgs
12 from .serve import main as serve_main
13 from .serve_sla import SweepServeSLAArgs
@@ -15,6 +17,7 @@ SUBCOMMANDS = (
17 (SweepServeArgs, serve_main),
18 (SweepServeSLAArgs, serve_sla_main),
19 (SweepPlotArgs, plot_main),
20 + (SweepPlotParetoArgs, plot_pareto_main),
21 )
22
23
vllm/benchmarks/sweep/plot_pareto.py new
+393
@@ -0,0 +1,393 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import argparse
4 +import math
5 +from concurrent.futures import ProcessPoolExecutor
6 +from dataclasses import dataclass
7 +from functools import partial
8 +from pathlib import Path
9 +from typing import ClassVar
10 +
11 +from vllm.utils.collection_utils import full_groupby
12 +from vllm.utils.import_utils import PlaceholderModule
13 +
14 +from .plot import DummyExecutor, _json_load_bytes
15 +from .utils import sanitize_filename
16 +
17 +try:
18 + import matplotlib.pyplot as plt
19 + import pandas as pd
20 + import seaborn as sns
21 +except ImportError:
22 + plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot")
23 + pd = PlaceholderModule("pandas")
24 + sns = PlaceholderModule("seaborn")
25 +
26 +
27 +def _first_present(run_data: dict[str, object], keys: list[str]):
28 + for key in keys:
29 + for candidate in {key, key.replace("_", "-"), key.replace("-", "_")}:
30 + if candidate in run_data:
31 + return run_data[candidate]
32 + return None
33 +
34 +
35 +def _get_numeric(
36 + run_data: dict[str, object],
37 + keys: list[str],
38 + *,
39 + allow_zero: bool = True,
40 +) -> float | None:
41 + value = _first_present(run_data, keys)
42 + if value is None:
43 + return None
44 +
45 + try:
46 + numeric = float(value)
47 + except (TypeError, ValueError) as exc:
48 + raise ValueError(
49 + f"Expected numeric value for one of {keys}, "
50 + f"but found {value!r} in {run_data=}"
51 + ) from exc
52 +
53 + if not allow_zero and numeric == 0:
54 + return None
55 +
56 + return numeric
57 +
58 +
59 +def _infer_user_count(
60 + run_data: dict[str, object],
61 + user_count_var: str | None,
62 +) -> float | None:
63 + candidates = [user_count_var] if user_count_var else []
64 + candidates.extend(["request_rate"])
65 + user_count = _get_numeric(run_data, candidates, allow_zero=False)
66 + if user_count is not None:
67 + return user_count
68 +
69 + # Fallback to the observed peak if configured value is missing.
70 + return _get_numeric(run_data, ["max_concurrent_requests"], allow_zero=False)
71 +
72 +
73 +def _infer_gpu_count(
74 + run_data: dict[str, object],
75 + gpu_count_var: str | None,
76 +) -> float:
77 + direct_candidates = [gpu_count_var] if gpu_count_var else []
78 + direct_gpu_count = _get_numeric(run_data, direct_candidates, allow_zero=False)
79 + if direct_gpu_count:
80 + return direct_gpu_count
81 +
82 + tp_size = _get_numeric(run_data, ["tensor_parallel_size", "tp"])
83 + pp_size = _get_numeric(run_data, ["pipeline_parallel_size", "pp"])
84 + dp_size = _get_numeric(run_data, ["data_parallel_size", "dp"])
85 + world_size = 1.0
86 + if tp_size:
87 + world_size *= tp_size
88 + if pp_size:
89 + world_size *= pp_size
90 + if dp_size:
91 + world_size *= dp_size
92 +
93 + return world_size
94 +
95 +
96 +def _get_throughput(
97 + run_data: dict[str, object],
98 + throughput_var: str,
99 +) -> float:
100 + throughput = _get_numeric(run_data, [throughput_var])
101 + if throughput is None:
102 + raise ValueError(
103 + f"Cannot find throughput metric {throughput_var!r} in run data. "
104 + f"Available keys: {sorted(run_data)}"
105 + )
106 +
107 + return throughput
108 +
109 +
110 +def _prepare_records(
111 + all_data: list[dict[str, object]],
112 + *,
113 + user_count_var: str | None,
114 + gpu_count_var: str | None,
115 +) -> tuple[list[dict[str, object]], int]:
116 + prepared = []
117 + skipped_missing_users = 0
118 +
119 + for record in all_data:
120 + throughput = _get_throughput(record, "output_throughput")
121 + user_count = _infer_user_count(record, user_count_var)
122 + if user_count is None:
123 + skipped_missing_users += 1
124 + continue
125 +
126 + gpu_count = _infer_gpu_count(record, gpu_count_var)
127 + tokens_per_user = throughput / user_count
128 + tokens_per_gpu = throughput / gpu_count
129 +
130 + prepared.append(
131 + {
132 + **record,
133 + "tokens_per_user": tokens_per_user,
134 + "tokens_per_gpu": tokens_per_gpu,
135 + "user_count_estimate": user_count,
136 + "gpu_count": gpu_count,
137 + }
138 + )
139 +
140 + return prepared, skipped_missing_users
141 +
142 +
143 +def _pareto_frontier(
144 + df: "pd.DataFrame",
145 + x_col: str,
146 + y_col: str,
147 + *,
148 + epsilon: float = 1e-9,
149 +) -> "pd.DataFrame":
150 + sorted_df = df.sort_values([x_col, y_col], ascending=[False, False])
151 + frontier_indices = []
152 + best_y = -math.inf
153 +
154 + for idx, row in sorted_df.iterrows():
155 + y_val = row[y_col]
156 + if y_val >= best_y - epsilon:
157 + frontier_indices.append(idx)
158 + best_y = max(best_y, y_val)
159 +
160 + return df.loc[frontier_indices]
161 +
162 +
163 +def _get_fig_path(
164 + fig_dir: Path,
165 + fig_group: tuple[tuple[str, str], ...],
166 +) -> Path:
167 + parts = ["PARETO"]
168 + if fig_group:
169 + parts.extend(f"{k}={v}" for k, v in fig_group)
170 + filename = sanitize_filename("-".join(parts) + ".png")
171 + return fig_dir / filename
172 +
173 +
174 +def _plot_fig(
175 + fig_dir: Path,
176 + fig_group_data: tuple[tuple[tuple[str, str], ...], list[dict[str, object]]],
177 + label_by: list[str],
178 + *,
179 + dry_run: bool,
180 +):
181 + fig_group, fig_data = fig_group_data
182 + fig_path = _get_fig_path(fig_dir, fig_group)
183 +
184 + print("[BEGIN FIGURE]")
185 + print(f"Group: {dict(fig_group)}")
186 + print(f"Output file: {fig_path}")
187 +
188 + if dry_run:
189 + print("[END FIGURE]")
190 + return
191 +
192 + df = pd.DataFrame.from_records(fig_data)
193 + df = df.dropna(subset=["tokens_per_user", "tokens_per_gpu"])
194 +
195 + if df.empty:
196 + print("No data points available after filtering; skipping.")
197 + print("[END FIGURE]")
198 + return
199 +
200 + frontier = _pareto_frontier(df, "tokens_per_user", "tokens_per_gpu")
201 + frontier = frontier.sort_values("tokens_per_user")
202 +
203 + fig, ax = plt.subplots()
204 + sns.scatterplot(
205 + data=df,
206 + x="tokens_per_user",
207 + y="tokens_per_gpu",
208 + color="0.5",
209 + alpha=0.6,
210 + ax=ax,
211 + label="All runs",
212 + )
213 + sns.lineplot(
214 + data=frontier,
215 + x="tokens_per_user",
216 + y="tokens_per_gpu",
217 + marker="o",
218 + ax=ax,
219 + label="Pareto frontier",
220 + )
221 +
222 + if label_by:
223 + for _, row in frontier.iterrows():
224 + label_parts = []
225 + for key in label_by:
226 + if key in row:
227 + label_parts.append(f"{key}={row[key]}")
228 + if label_parts:
229 + ax.text(
230 + row["tokens_per_user"],
231 + row["tokens_per_gpu"],
232 + "\n".join(label_parts),
233 + fontsize=8,
234 + )
235 +
236 + ax.set_xlabel("Tokens/s/user")
237 + ax.set_ylabel("Tokens/s/GPU")
238 + ax.grid(True, linestyle="--", linewidth=0.5, alpha=0.6)
239 + fig.tight_layout()
240 + fig.savefig(fig_path)
241 + plt.close(fig)
242 +
243 + print(
244 + f"Plotted {len(df)} points; Pareto frontier size: {len(frontier)}.",
245 + )
246 + print("[END FIGURE]")
247 +
248 +
249 +def plot_pareto(
250 + output_dir: Path,
251 + user_count_var: str | None,
252 + gpu_count_var: str | None,
253 + label_by: list[str],
254 + *,
255 + dry_run: bool,
256 +):
257 + fig_dir = output_dir / "pareto"
258 + raw_data = [
259 + run_data
260 + for path in output_dir.rglob("**/summary.json")
261 + for run_data in _json_load_bytes(path)
262 + ]
263 +
264 + if not raw_data:
265 + raise ValueError(f"Did not find any parameter sweep results under {output_dir}")
266 +
267 + fig_dir.mkdir(parents=True, exist_ok=True)
268 +
269 + prepared_data, skipped_missing_users = _prepare_records(
270 + raw_data,
271 + user_count_var=user_count_var,
272 + gpu_count_var=gpu_count_var,
273 + )
274 +
275 + if skipped_missing_users:
276 + print(
277 + f"Skipped {skipped_missing_users} runs without a user count "
278 + "(`max_concurrency` or `max_concurrent_requests`).",
279 + )
280 +
281 + if not prepared_data:
282 + raise ValueError(
283 + "No data points with both throughput and user count available "
284 + "to plot Pareto frontier.",
285 + )
286 +
287 + fig_groups = full_groupby(
288 + prepared_data,
289 + key=lambda item: tuple(),
290 + )
291 +
292 + with DummyExecutor() if len(fig_groups) <= 1 else ProcessPoolExecutor() as executor:
293 + all(
294 + executor.map(
295 + partial(
296 + _plot_fig,
297 + fig_dir,
298 + label_by=label_by,
299 + dry_run=dry_run,
300 + ),
301 + fig_groups,
302 + )
303 + )
304 +
305 +
306 +@dataclass
307 +class SweepPlotParetoArgs:
308 + output_dir: Path
309 + user_count_var: str | None
310 + gpu_count_var: str | None
311 + label_by: list[str]
312 + dry_run: bool
313 +
314 + parser_name: ClassVar[str] = "plot_pareto"
315 + parser_help: ClassVar[str] = (
316 + "Plot Pareto frontier between tokens/s/user and tokens/s/GPU "
317 + "from parameter sweep results."
318 + )
319 +
320 + @classmethod
321 + def from_cli_args(cls, args: argparse.Namespace):
322 + output_dir = Path(args.OUTPUT_DIR)
323 + if not output_dir.exists():
324 + raise ValueError(f"No parameter sweep results under {output_dir}")
325 +
326 + label_by = [] if not args.label_by else args.label_by.split(",")
327 +
328 + return cls(
329 + output_dir=output_dir,
330 + user_count_var=args.user_count_var,
331 + gpu_count_var=args.gpu_count_var,
332 + label_by=label_by,
333 + dry_run=args.dry_run,
334 + )
335 +
336 + @classmethod
337 + def add_cli_args(cls, parser: argparse.ArgumentParser):
338 + parser.add_argument(
339 + "OUTPUT_DIR",
340 + type=str,
341 + default="results",
342 + help="The directory containing the sweep results to plot.",
343 + )
344 + parser.add_argument(
345 + "--user-count-var",
346 + type=str,
347 + default="max_concurrency",
348 + help="Result key that stores concurrent user count. "
349 + "Falls back to max_concurrent_requests if missing.",
350 + )
351 + parser.add_argument(
352 + "--gpu-count-var",
353 + type=str,
354 + default=None,
355 + help="Result key that stores GPU count. "
356 + "If not provided, falls back to num_gpus/gpu_count "
357 + "or tensor_parallel_size * pipeline_parallel_size.",
358 + )
359 + parser.add_argument(
360 + "--label-by",
361 + type=str,
362 + default="max_concurrency,gpu_count",
363 + help="Comma-separated list of fields to annotate on Pareto frontier "
364 + "points.",
365 + )
366 + parser.add_argument(
367 + "--dry-run",
368 + action="store_true",
369 + help="If set, prints the figures to plot without drawing them.",
370 + )
371 +
372 + return parser
373 +
374 +
375 +def run_main(args: SweepPlotParetoArgs):
376 + return plot_pareto(
377 + output_dir=args.output_dir,
378 + user_count_var=args.user_count_var,
379 + gpu_count_var=args.gpu_count_var,
380 + label_by=args.label_by,
381 + dry_run=args.dry_run,
382 + )
383 +
384 +
385 +def main(args: argparse.Namespace):
386 + run_main(SweepPlotParetoArgs.from_cli_args(args))
387 +
388 +
389 +if __name__ == "__main__":
390 + parser = argparse.ArgumentParser(description=SweepPlotParetoArgs.parser_help)
391 + SweepPlotParetoArgs.add_cli_args(parser)
392 +
393 + main(parser.parse_args())