A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 04:15 UTC
[Frontend] Add `vllm bench sweep` to CLI (#27639)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Cyrus Leung committed
Oct 29, 2025 at 20:59 UTC
ecca3fee761c9dd710daf3acb2e646d10fb631d7
19 files changed
+340
-168
docs/cli/.nav.yml
+1
-1
@@ -5,4 +5,4 @@ nav:
5
- complete.md
6
- run-batch.md
7
- vllm bench:
8
- - bench/*.md
8
+ - bench/**/*.md
docs/cli/bench/sweep/plot.md
new
+9
@@ -0,0 +1,9 @@
1
+# vllm bench sweep plot
2
+
3
+## JSON CLI Arguments
4
+
5
+--8<-- "docs/cli/json_tip.inc.md"
6
+
7
+## Options
8
+
9
+--8<-- "docs/argparse/bench_sweep_plot.md"
docs/cli/bench/sweep/serve.md
new
+9
@@ -0,0 +1,9 @@
1
+# vllm bench sweep serve
2
+
3
+## JSON CLI Arguments
4
+
5
+--8<-- "docs/cli/json_tip.inc.md"
6
+
7
+## Options
8
+
9
+--8<-- "docs/argparse/bench_sweep_serve.md"
docs/cli/bench/sweep/serve_sla.md
new
+9
@@ -0,0 +1,9 @@
1
+# vllm bench sweep serve_sla
2
+
3
+## JSON CLI Arguments
4
+
5
+--8<-- "docs/cli/json_tip.inc.md"
6
+
7
+## Options
8
+
9
+--8<-- "docs/argparse/bench_sweep_serve_sla.md"
docs/contributing/benchmarks.md
+3
-3
@@ -1061,7 +1061,7 @@ Follow these steps to run the script:
1061
Example command:
1062
1063
```bash
1064
-python -m vllm.benchmarks.sweep.serve \
1064
+vllm bench sweep serve \
1065
--serve-cmd 'vllm serve meta-llama/Llama-2-7b-chat-hf' \
1066
--bench-cmd 'vllm bench serve --model meta-llama/Llama-2-7b-chat-hf --backend vllm --endpoint /v1/completions --dataset-name sharegpt --dataset-path benchmarks/ShareGPT_V3_unfiltered_cleaned_split.json' \
1067
--serve-params benchmarks/serve_hparams.json \
@@ -1109,7 +1109,7 @@ For example, to ensure E2E latency within different target values for 99% of req
1109
Example command:
1110
1111
```bash
1112
-python -m vllm.benchmarks.sweep.serve_sla \
1112
+vllm bench sweep serve_sla \
1113
--serve-cmd 'vllm serve meta-llama/Llama-2-7b-chat-hf' \
1114
--bench-cmd 'vllm bench serve --model meta-llama/Llama-2-7b-chat-hf --backend vllm --endpoint /v1/completions --dataset-name sharegpt --dataset-path benchmarks/ShareGPT_V3_unfiltered_cleaned_split.json' \
1115
--serve-params benchmarks/serve_hparams.json \
@@ -1138,7 +1138,7 @@ The algorithm for adjusting the SLA variable is as follows:
1138
Example command:
1139
1140
```bash
1141
-python -m vllm.benchmarks.sweep.plot benchmarks/results/<timestamp> \
1141
+vllm bench sweep plot benchmarks/results/<timestamp> \
1142
--var-x max_concurrency \
1143
--row-by random_input_len \
1144
--col-by random_output_len \
docs/mkdocs/hooks/generate_argparse.py
+24
-10
@@ -56,15 +56,20 @@ def auto_mock(module, attr, max_mocks=50):
56
)
57
58
59
-latency = auto_mock("vllm.benchmarks", "latency")
60
-serve = auto_mock("vllm.benchmarks", "serve")
61
-throughput = auto_mock("vllm.benchmarks", "throughput")
59
+bench_latency = auto_mock("vllm.benchmarks", "latency")
60
+bench_serve = auto_mock("vllm.benchmarks", "serve")
61
+bench_sweep_plot = auto_mock("vllm.benchmarks.sweep.plot", "SweepPlotArgs")
62
+bench_sweep_serve = auto_mock("vllm.benchmarks.sweep.serve", "SweepServeArgs")
63
+bench_sweep_serve_sla = auto_mock(
64
+ "vllm.benchmarks.sweep.serve_sla", "SweepServeSLAArgs"
65
+)
66
+bench_throughput = auto_mock("vllm.benchmarks", "throughput")
67
AsyncEngineArgs = auto_mock("vllm.engine.arg_utils", "AsyncEngineArgs")
68
EngineArgs = auto_mock("vllm.engine.arg_utils", "EngineArgs")
69
ChatCommand = auto_mock("vllm.entrypoints.cli.openai", "ChatCommand")
70
CompleteCommand = auto_mock("vllm.entrypoints.cli.openai", "CompleteCommand")
66
-cli_args = auto_mock("vllm.entrypoints.openai", "cli_args")
67
-run_batch = auto_mock("vllm.entrypoints.openai", "run_batch")
71
+openai_cli_args = auto_mock("vllm.entrypoints.openai", "cli_args")
72
+openai_run_batch = auto_mock("vllm.entrypoints.openai", "run_batch")
73
FlexibleArgumentParser = auto_mock(
74
"vllm.utils.argparse_utils", "FlexibleArgumentParser"
75
)
@@ -114,6 +119,9 @@ class MarkdownFormatter(HelpFormatter):
119
self._markdown_output.append(f"{action.help}\n\n")
120
121
if (default := action.default) != SUPPRESS:
122
+ # Make empty string defaults visible
123
+ if default == "":
124
+ default = '""'
125
self._markdown_output.append(f"Default: `{default}`\n\n")
126
127
def format_help(self):
@@ -150,17 +158,23 @@ def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
158
159
# Create parsers to document
160
parsers = {
161
+ # Engine args
162
"engine_args": create_parser(EngineArgs.add_cli_args),
163
"async_engine_args": create_parser(
164
AsyncEngineArgs.add_cli_args, async_args_only=True
165
),
157
- "serve": create_parser(cli_args.make_arg_parser),
166
+ # CLI
167
+ "serve": create_parser(openai_cli_args.make_arg_parser),
168
"chat": create_parser(ChatCommand.add_cli_args),
169
"complete": create_parser(CompleteCommand.add_cli_args),
160
- "bench_latency": create_parser(latency.add_cli_args),
161
- "bench_throughput": create_parser(throughput.add_cli_args),
162
- "bench_serve": create_parser(serve.add_cli_args),
163
- "run-batch": create_parser(run_batch.make_arg_parser),
170
+ "run-batch": create_parser(openai_run_batch.make_arg_parser),
171
+ # Benchmark CLI
172
+ "bench_latency": create_parser(bench_latency.add_cli_args),
173
+ "bench_serve": create_parser(bench_serve.add_cli_args),
174
+ "bench_sweep_plot": create_parser(bench_sweep_plot.add_cli_args),
175
+ "bench_sweep_serve": create_parser(bench_sweep_serve.add_cli_args),
176
+ "bench_sweep_serve_sla": create_parser(bench_sweep_serve_sla.add_cli_args),
177
+ "bench_throughput": create_parser(bench_throughput.add_cli_args),
178
}
179
180
# Generate documentation for each parser
setup.py
+1
-1
@@ -709,7 +709,7 @@ setup(
709
ext_modules=ext_modules,
710
install_requires=get_requirements(),
711
extras_require={
712
- "bench": ["pandas", "datasets"],
712
+ "bench": ["pandas", "matplotlib", "seaborn", "datasets"],
713
"tensorizer": ["tensorizer==2.10.1"],
714
"fastsafetensors": ["fastsafetensors >= 0.1.10"],
715
"runai": ["runai-model-streamer[s3,gcs] >= 0.14.0"],
tools/profiler/visualize_layerwise_profile.py
+6
-6
@@ -141,7 +141,7 @@ def attempt_to_make_names_unique(entries_and_traces):
141
"""
142
143
144
-def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
144
+def group_trace_by_operations(trace_df: "pd.DataFrame") -> "pd.DataFrame":
145
def is_rms_norm(op_name: str):
146
if "rms_norm_kernel" in op_name:
147
return True
@@ -370,12 +370,12 @@ def group_trace_by_operations(trace_df: pd.DataFrame) -> pd.DataFrame:
370
371
372
def plot_trace_df(
373
- traces_df: pd.DataFrame,
373
+ traces_df: "pd.DataFrame",
374
plot_metric: str,
375
plot_title: str,
376
output: Path | None = None,
377
):
378
- def get_phase_description(traces_df: pd.DataFrame, phase: str) -> str:
378
+ def get_phase_description(traces_df: "pd.DataFrame", phase: str) -> str:
379
phase_df = traces_df.query(f'phase == "{phase}"')
380
descs = phase_df["phase_desc"].to_list()
381
assert all([desc == descs[0] for desc in descs])
@@ -438,7 +438,7 @@ def main(
438
top_k: int,
439
json_nodes_to_fold: list[str],
440
):
441
- def prepare_data(profile_json: dict, step_keys: list[str]) -> pd.DataFrame:
441
+ def prepare_data(profile_json: dict, step_keys: list[str]) -> "pd.DataFrame":
442
def get_entries_and_traces(key: str):
443
entries_and_traces: list[tuple[Any, Any]] = []
444
for root in profile_json[key]["summary_stats"]:
@@ -449,8 +449,8 @@ def main(
449
return entries_and_traces
450
451
def keep_only_top_entries(
452
- df: pd.DataFrame, metric: str, top_k: int = 9
453
- ) -> pd.DataFrame:
452
+ df: "pd.DataFrame", metric: str, top_k: int = 9
453
+ ) -> "pd.DataFrame":
454
df.loc[df.nsmallest(len(df) - top_k + 1, metric).index, ["name"]] = "others"
455
return df
456
vllm/benchmarks/sweep/cli.py
new
+38
@@ -0,0 +1,38 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import argparse
4
+
5
+from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
6
+
7
+from .plot import SweepPlotArgs
8
+from .plot import main as plot_main
9
+from .serve import SweepServeArgs
10
+from .serve import main as serve_main
11
+from .serve_sla import SweepServeSLAArgs
12
+from .serve_sla import main as serve_sla_main
13
+
14
+SUBCOMMANDS = (
15
+ (SweepServeArgs, serve_main),
16
+ (SweepServeSLAArgs, serve_sla_main),
17
+ (SweepPlotArgs, plot_main),
18
+)
19
+
20
+
21
+def add_cli_args(parser: argparse.ArgumentParser):
22
+ subparsers = parser.add_subparsers(required=True, dest="sweep_type")
23
+
24
+ for cmd, entrypoint in SUBCOMMANDS:
25
+ cmd_subparser = subparsers.add_parser(
26
+ cmd.parser_name,
27
+ description=cmd.parser_help,
28
+ usage=f"vllm bench sweep {cmd.parser_name} [options]",
29
+ )
30
+ cmd_subparser.set_defaults(dispatch_function=entrypoint)
31
+ cmd.add_cli_args(cmd_subparser)
32
+ cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
33
+ subcmd=f"sweep {cmd.parser_name}"
34
+ )
35
+
36
+
37
+def main(args: argparse.Namespace):
38
+ args.dispatch_function(args)
vllm/benchmarks/sweep/plot.py
+181
-131
@@ -8,16 +8,24 @@ from dataclasses import dataclass
8
from functools import partial
9
from pathlib import Path
10
from types import TracebackType
11
+from typing import ClassVar
12
12
-import matplotlib.pyplot as plt
13
-import pandas as pd
14
-import seaborn as sns
13
from typing_extensions import Self, override
14
15
from vllm.utils.collection_utils import full_groupby
16
+from vllm.utils.import_utils import PlaceholderModule
17
18
from .utils import sanitize_filename
19
20
+try:
21
+ import matplotlib.pyplot as plt
22
+ import pandas as pd
23
+ import seaborn as sns
24
+except ImportError:
25
+ plt = PlaceholderModule("matplotlib").placeholder_attr("pyplot")
26
+ pd = PlaceholderModule("pandas")
27
+ seaborn = PlaceholderModule("seaborn")
28
+
29
30
@dataclass
31
class PlotFilterBase(ABC):
@@ -40,7 +48,7 @@ class PlotFilterBase(ABC):
48
)
49
50
@abstractmethod
43
- def apply(self, df: pd.DataFrame) -> pd.DataFrame:
51
+ def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
52
"""Applies this filter to a DataFrame."""
53
raise NotImplementedError
54
@@ -48,7 +56,7 @@ class PlotFilterBase(ABC):
56
@dataclass
57
class PlotEqualTo(PlotFilterBase):
58
@override
51
- def apply(self, df: pd.DataFrame) -> pd.DataFrame:
59
+ def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
60
try:
61
target = float(self.target)
62
except ValueError:
@@ -60,28 +68,28 @@ class PlotEqualTo(PlotFilterBase):
68
@dataclass
69
class PlotLessThan(PlotFilterBase):
70
@override
63
- def apply(self, df: pd.DataFrame) -> pd.DataFrame:
71
+ def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
72
return df[df[self.var] < float(self.target)]
73
74
75
@dataclass
76
class PlotLessThanOrEqualTo(PlotFilterBase):
77
@override
70
- def apply(self, df: pd.DataFrame) -> pd.DataFrame:
78
+ def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
79
return df[df[self.var] <= float(self.target)]
80
81
82
@dataclass
83
class PlotGreaterThan(PlotFilterBase):
84
@override
77
- def apply(self, df: pd.DataFrame) -> pd.DataFrame:
85
+ def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
86
return df[df[self.var] > float(self.target)]
87
88
89
@dataclass
90
class PlotGreaterThanOrEqualTo(PlotFilterBase):
91
@override
84
- def apply(self, df: pd.DataFrame) -> pd.DataFrame:
92
+ def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
93
return df[df[self.var] >= float(self.target)]
94
95
@@ -103,7 +111,7 @@ class PlotFilters(list[PlotFilterBase]):
111
112
return cls(PlotFilterBase.parse_str(e) for e in s.split(","))
113
106
- def apply(self, df: pd.DataFrame) -> pd.DataFrame:
114
+ def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
115
for item in self:
116
df = item.apply(df)
117
@@ -127,7 +135,7 @@ class PlotBinner:
135
f"Valid operators are: {sorted(PLOT_BINNERS)}",
136
)
137
130
- def apply(self, df: pd.DataFrame) -> pd.DataFrame:
138
+ def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
139
"""Applies this binner to a DataFrame."""
140
df = df.copy()
141
df[self.var] = df[self.var] // self.bin_size * self.bin_size
@@ -147,7 +155,7 @@ class PlotBinners(list[PlotBinner]):
155
156
return cls(PlotBinner.parse_str(e) for e in s.split(","))
157
150
- def apply(self, df: pd.DataFrame) -> pd.DataFrame:
158
+ def apply(self, df: "pd.DataFrame") -> "pd.DataFrame":
159
for item in self:
160
df = item.apply(df)
161
@@ -396,135 +404,177 @@ def plot(
404
)
405
406
399
-def add_cli_args(parser: argparse.ArgumentParser):
400
- parser.add_argument(
401
- "OUTPUT_DIR",
402
- type=str,
403
- default="results",
404
- help="The directory containing the results to plot, "
405
- "i.e., the `--output-dir` argument to the parameter sweep script.",
406
- )
407
- parser.add_argument(
408
- "--fig-dir",
409
- type=str,
410
- default="",
411
- help="The directory to save the figures, relative to `OUTPUT_DIR`. "
412
- "By default, the same directory is used.",
413
- )
414
- parser.add_argument(
415
- "--fig-by",
416
- type=str,
417
- default="",
418
- help="A comma-separated list of variables, such that a separate figure "
419
- "is created for each combination of these variables.",
420
- )
421
- parser.add_argument(
422
- "--row-by",
423
- type=str,
424
- default="",
425
- help="A comma-separated list of variables, such that a separate row "
426
- "is created for each combination of these variables.",
427
- )
428
- parser.add_argument(
429
- "--col-by",
430
- type=str,
431
- default="",
432
- help="A comma-separated list of variables, such that a separate column "
433
- "is created for each combination of these variables.",
434
- )
435
- parser.add_argument(
436
- "--curve-by",
437
- type=str,
438
- default=None,
439
- help="A comma-separated list of variables, such that a separate curve "
440
- "is created for each combination of these variables.",
441
- )
442
- parser.add_argument(
443
- "--var-x",
444
- type=str,
445
- default="request_throughput",
446
- help="The variable for the x-axis.",
447
- )
448
- parser.add_argument(
449
- "--var-y",
450
- type=str,
451
- default="p99_e2el_ms",
452
- help="The variable for the y-axis",
453
- )
454
- parser.add_argument(
455
- "--filter-by",
456
- type=str,
457
- default="",
458
- help="A comma-separated list of statements indicating values to filter by. "
459
- "This is useful to remove outliers. "
460
- "Example: `max_concurrency<1000,max_num_batched_tokens<=4096` means "
461
- "plot only the points where `max_concurrency` is less than 1000 and "
462
- "`max_num_batched_tokens` is no greater than 4096.",
463
- )
464
- parser.add_argument(
465
- "--bin-by",
466
- type=str,
467
- default="",
468
- help="A comma-separated list of statements indicating values to bin by. "
469
- "This is useful to avoid plotting points that are too close together. "
470
- "Example: `request_throughput%1` means "
471
- "use a bin size of 1 for the `request_throughput` variable.",
472
- )
473
- parser.add_argument(
474
- "--scale-x",
475
- type=str,
476
- default=None,
477
- help="The scale to use for the x-axis. "
478
- "Currently only accepts string values such as 'log' and 'sqrt'. "
479
- "See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html",
480
- )
481
- parser.add_argument(
482
- "--scale-y",
483
- type=str,
484
- default=None,
485
- help="The scale to use for the y-axis. "
486
- "Currently only accepts string values such as 'log' and 'sqrt'. "
487
- "See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html",
488
- )
489
- parser.add_argument(
490
- "--dry-run",
491
- action="store_true",
492
- help="If set, prints the information about each figure to plot, "
493
- "then exits without drawing them.",
494
- )
407
+@dataclass
408
+class SweepPlotArgs:
409
+ output_dir: Path
410
+ fig_dir: Path
411
+ fig_by: list[str]
412
+ row_by: list[str]
413
+ col_by: list[str]
414
+ curve_by: list[str]
415
+ var_x: str
416
+ var_y: str
417
+ filter_by: PlotFilters
418
+ bin_by: PlotBinners
419
+ scale_x: str | None
420
+ scale_y: str | None
421
+ dry_run: bool
422
+
423
+ parser_name: ClassVar[str] = "plot"
424
+ parser_help: ClassVar[str] = "Plot performance curves from parameter sweep results."
425
426
+ @classmethod
427
+ def from_cli_args(cls, args: argparse.Namespace):
428
+ output_dir = Path(args.OUTPUT_DIR)
429
+ if not output_dir.exists():
430
+ raise ValueError(f"No parameter sweep results under {output_dir}")
431
+
432
+ curve_by = [] if not args.curve_by else args.curve_by.split(",")
433
+ row_by = [] if not args.row_by else args.row_by.split(",")
434
+ col_by = [] if not args.col_by else args.col_by.split(",")
435
+ fig_by = [] if not args.fig_by else args.fig_by.split(",")
436
+
437
+ return cls(
438
+ output_dir=output_dir,
439
+ fig_dir=output_dir / args.fig_dir,
440
+ fig_by=fig_by,
441
+ row_by=row_by,
442
+ col_by=col_by,
443
+ curve_by=curve_by,
444
+ var_x=args.var_x,
445
+ var_y=args.var_y,
446
+ filter_by=PlotFilters.parse_str(args.filter_by),
447
+ bin_by=PlotBinners.parse_str(args.bin_by),
448
+ scale_x=args.scale_x,
449
+ scale_y=args.scale_y,
450
+ dry_run=args.dry_run,
451
+ )
452
497
-def main(args: argparse.Namespace):
498
- output_dir = Path(args.OUTPUT_DIR)
499
- if not output_dir.exists():
500
- raise ValueError(f"No parameter sweep results under {output_dir}")
501
-
502
- curve_by = [] if not args.curve_by else args.curve_by.split(",")
503
- row_by = [] if not args.row_by else args.row_by.split(",")
504
- col_by = [] if not args.col_by else args.col_by.split(",")
505
- fig_by = [] if not args.fig_by else args.fig_by.split(",")
506
-
507
- plot(
508
- output_dir=output_dir,
509
- fig_dir=output_dir / args.fig_dir,
510
- fig_by=fig_by,
511
- row_by=row_by,
512
- col_by=col_by,
513
- curve_by=curve_by,
453
+ @classmethod
454
+ def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
455
+ parser.add_argument(
456
+ "OUTPUT_DIR",
457
+ type=str,
458
+ default="results",
459
+ help="The directory containing the results to plot, "
460
+ "i.e., the `--output-dir` argument to the parameter sweep script.",
461
+ )
462
+ parser.add_argument(
463
+ "--fig-dir",
464
+ type=str,
465
+ default="",
466
+ help="The directory to save the figures, relative to `OUTPUT_DIR`. "
467
+ "By default, the same directory is used.",
468
+ )
469
+ parser.add_argument(
470
+ "--fig-by",
471
+ type=str,
472
+ default="",
473
+ help="A comma-separated list of variables, such that a separate figure "
474
+ "is created for each combination of these variables.",
475
+ )
476
+ parser.add_argument(
477
+ "--row-by",
478
+ type=str,
479
+ default="",
480
+ help="A comma-separated list of variables, such that a separate row "
481
+ "is created for each combination of these variables.",
482
+ )
483
+ parser.add_argument(
484
+ "--col-by",
485
+ type=str,
486
+ default="",
487
+ help="A comma-separated list of variables, such that a separate column "
488
+ "is created for each combination of these variables.",
489
+ )
490
+ parser.add_argument(
491
+ "--curve-by",
492
+ type=str,
493
+ default=None,
494
+ help="A comma-separated list of variables, such that a separate curve "
495
+ "is created for each combination of these variables.",
496
+ )
497
+ parser.add_argument(
498
+ "--var-x",
499
+ type=str,
500
+ default="request_throughput",
501
+ help="The variable for the x-axis.",
502
+ )
503
+ parser.add_argument(
504
+ "--var-y",
505
+ type=str,
506
+ default="p99_e2el_ms",
507
+ help="The variable for the y-axis",
508
+ )
509
+ parser.add_argument(
510
+ "--filter-by",
511
+ type=str,
512
+ default="",
513
+ help="A comma-separated list of statements indicating values to filter by. "
514
+ "This is useful to remove outliers. "
515
+ "Example: `max_concurrency<1000,max_num_batched_tokens<=4096` means "
516
+ "plot only the points where `max_concurrency` is less than 1000 and "
517
+ "`max_num_batched_tokens` is no greater than 4096.",
518
+ )
519
+ parser.add_argument(
520
+ "--bin-by",
521
+ type=str,
522
+ default="",
523
+ help="A comma-separated list of statements indicating values to bin by. "
524
+ "This is useful to avoid plotting points that are too close together. "
525
+ "Example: `request_throughput%%1` means "
526
+ "use a bin size of 1 for the `request_throughput` variable.",
527
+ )
528
+ parser.add_argument(
529
+ "--scale-x",
530
+ type=str,
531
+ default=None,
532
+ help="The scale to use for the x-axis. "
533
+ "Currently only accepts string values such as 'log' and 'sqrt'. "
534
+ "See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html",
535
+ )
536
+ parser.add_argument(
537
+ "--scale-y",
538
+ type=str,
539
+ default=None,
540
+ help="The scale to use for the y-axis. "
541
+ "Currently only accepts string values such as 'log' and 'sqrt'. "
542
+ "See also: https://seaborn.pydata.org/generated/seaborn.objects.Plot.scale.html",
543
+ )
544
+ parser.add_argument(
545
+ "--dry-run",
546
+ action="store_true",
547
+ help="If set, prints the information about each figure to plot, "
548
+ "then exits without drawing them.",
549
+ )
550
+
551
+ return parser
552
+
553
+
554
+def run_main(args: SweepPlotArgs):
555
+ return plot(
556
+ output_dir=args.output_dir,
557
+ fig_dir=args.fig_dir,
558
+ fig_by=args.fig_by,
559
+ row_by=args.row_by,
560
+ col_by=args.col_by,
561
+ curve_by=args.curve_by,
562
var_x=args.var_x,
563
var_y=args.var_y,
516
- filter_by=PlotFilters.parse_str(args.filter_by),
517
- bin_by=PlotBinners.parse_str(args.bin_by),
564
+ filter_by=args.filter_by,
565
+ bin_by=args.bin_by,
566
scale_x=args.scale_x,
567
scale_y=args.scale_y,
568
dry_run=args.dry_run,
569
)
570
571
572
+def main(args: argparse.Namespace):
573
+ run_main(SweepPlotArgs.from_cli_args(args))
574
+
575
+
576
if __name__ == "__main__":
525
- parser = argparse.ArgumentParser(
526
- description="Plot performance curves from parameter sweep results."
527
- )
528
- add_cli_args(parser)
577
+ parser = argparse.ArgumentParser(description=SweepPlotArgs.parser_help)
578
+ SweepPlotArgs.add_cli_args(parser)
579
580
main(parser.parse_args())
vllm/benchmarks/sweep/serve.py
+11
-4
@@ -7,13 +7,19 @@ import shlex
7
from dataclasses import dataclass
8
from datetime import datetime
9
from pathlib import Path
10
+from typing import ClassVar
11
11
-import pandas as pd
12
+from vllm.utils.import_utils import PlaceholderModule
13
14
from .param_sweep import ParameterSweep, ParameterSweepItem
15
from .server import ServerProcess
16
from .utils import sanitize_filename
17
18
+try:
19
+ import pandas as pd
20
+except ImportError:
21
+ pd = PlaceholderModule("pandas")
22
+
23
24
@contextlib.contextmanager
25
def run_server(
@@ -257,6 +263,9 @@ class SweepServeArgs:
263
dry_run: bool
264
resume: str | None
265
266
+ parser_name: ClassVar[str] = "serve"
267
+ parser_help: ClassVar[str] = "Run vLLM server benchmark under multiple settings."
268
+
269
@classmethod
270
def from_cli_args(cls, args: argparse.Namespace):
271
serve_cmd = shlex.split(args.serve_cmd)
@@ -401,9 +410,7 @@ def main(args: argparse.Namespace):
410
411
412
if __name__ == "__main__":
404
- parser = argparse.ArgumentParser(
405
- description="Run vLLM server benchmark under multiple settings."
406
- )
413
+ parser = argparse.ArgumentParser(description=SweepServeArgs.parser_help)
414
SweepServeArgs.add_cli_args(parser)
415
416
main(parser.parse_args())
vllm/benchmarks/sweep/serve_sla.py
+15
-7
@@ -7,17 +7,23 @@ import math
7
from dataclasses import asdict, dataclass
8
from datetime import datetime
9
from pathlib import Path
10
-from typing import Literal, get_args
10
+from typing import ClassVar, Literal, get_args
11
12
-import pandas as pd
12
from typing_extensions import assert_never
13
14
+from vllm.utils.import_utils import PlaceholderModule
15
+
16
from .param_sweep import ParameterSweep, ParameterSweepItem
17
from .serve import SweepServeArgs, run_benchmark, run_server
18
from .server import ServerProcess
19
from .sla_sweep import SLASweep, SLASweepItem
20
from .utils import sanitize_filename
21
22
+try:
23
+ import pandas as pd
24
+except ImportError:
25
+ pd = PlaceholderModule("pandas")
26
+
27
28
def _get_sla_base_path(
29
output_dir: Path,
@@ -399,6 +405,9 @@ class SweepServeSLAArgs(SweepServeArgs):
405
sla_params: SLASweep
406
sla_variable: SLAVariable
407
408
+ parser_name: ClassVar[str] = "serve_sla"
409
+ parser_help: ClassVar[str] = "Tune a variable to meet SLAs under multiple settings."
410
+
411
@classmethod
412
def from_cli_args(cls, args: argparse.Namespace):
413
# NOTE: Don't use super() as `from_cli_args` calls `cls()`
@@ -419,7 +428,8 @@ class SweepServeSLAArgs(SweepServeArgs):
428
def add_cli_args(cls, parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
429
parser = super().add_cli_args(parser)
430
422
- parser.add_argument(
431
+ sla_group = parser.add_argument_group("sla options")
432
+ sla_group.add_argument(
433
"--sla-params",
434
type=str,
435
required=True,
@@ -431,7 +441,7 @@ class SweepServeSLAArgs(SweepServeArgs):
441
"the maximum `sla_variable` that satisfies the constraints for "
442
"each combination of `serve_params`, `bench_params`, and `sla_params`.",
443
)
434
- parser.add_argument(
444
+ sla_group.add_argument(
445
"--sla-variable",
446
type=str,
447
choices=get_args(SLAVariable),
@@ -476,9 +486,7 @@ def main(args: argparse.Namespace):
486
487
488
if __name__ == "__main__":
479
- parser = argparse.ArgumentParser(
480
- description="Tune a variable to meet SLAs under multiple settings."
481
- )
489
+ parser = argparse.ArgumentParser(description=SweepServeSLAArgs.parser_help)
490
SweepServeSLAArgs.add_cli_args(parser)
491
492
main(parser.parse_args())
vllm/entrypoints/cli/__init__.py
+2
@@ -2,10 +2,12 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
from vllm.entrypoints.cli.benchmark.latency import BenchmarkLatencySubcommand
4
from vllm.entrypoints.cli.benchmark.serve import BenchmarkServingSubcommand
5
+from vllm.entrypoints.cli.benchmark.sweep import BenchmarkSweepSubcommand
6
from vllm.entrypoints.cli.benchmark.throughput import BenchmarkThroughputSubcommand
7
8
__all__: list[str] = [
9
"BenchmarkLatencySubcommand",
10
"BenchmarkServingSubcommand",
11
+ "BenchmarkSweepSubcommand",
12
"BenchmarkThroughputSubcommand",
13
]
vllm/entrypoints/cli/benchmark/base.py
+1
-1
@@ -6,7 +6,7 @@ from vllm.entrypoints.cli.types import CLISubcommand
6
7
8
class BenchmarkSubcommandBase(CLISubcommand):
9
- """The base class of subcommands for vllm bench."""
9
+ """The base class of subcommands for `vllm bench`."""
10
11
help: str
12
vllm/entrypoints/cli/benchmark/latency.py
+1
-1
@@ -7,7 +7,7 @@ from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
7
8
9
class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
10
- """The `latency` subcommand for vllm bench."""
10
+ """The `latency` subcommand for `vllm bench`."""
11
12
name = "latency"
13
help = "Benchmark the latency of a single batch of requests."
vllm/entrypoints/cli/benchmark/serve.py
+1
-1
@@ -7,7 +7,7 @@ from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
7
8
9
class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
10
- """The `serve` subcommand for vllm bench."""
10
+ """The `serve` subcommand for `vllm bench`."""
11
12
name = "serve"
13
help = "Benchmark the online serving throughput."
vllm/entrypoints/cli/benchmark/sweep.py
new
+21
@@ -0,0 +1,21 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import argparse
4
+
5
+from vllm.benchmarks.sweep.cli import add_cli_args, main
6
+from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
7
+
8
+
9
+class BenchmarkSweepSubcommand(BenchmarkSubcommandBase):
10
+ """The `sweep` subcommand for `vllm bench`."""
11
+
12
+ name = "sweep"
13
+ help = "Benchmark for a parameter sweep."
14
+
15
+ @classmethod
16
+ def add_cli_args(cls, parser: argparse.ArgumentParser) -> None:
17
+ add_cli_args(parser)
18
+
19
+ @staticmethod
20
+ def cmd(args: argparse.Namespace) -> None:
21
+ main(args)
vllm/entrypoints/cli/benchmark/throughput.py
+1
-1
@@ -7,7 +7,7 @@ from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
7
8
9
class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
10
- """The `throughput` subcommand for vllm bench."""
10
+ """The `throughput` subcommand for `vllm bench`."""
11
12
name = "throughput"
13
help = "Benchmark offline inference throughput."
vllm/profiler/layerwise_profile.py
+6
-1
@@ -7,7 +7,6 @@ from collections.abc import Callable
7
from dataclasses import asdict, dataclass, field
8
from typing import Any, Optional, TypeAlias
9
10
-import pandas as pd
10
from torch._C._autograd import DeviceType, _KinetoEvent, _ProfilerResult
11
from torch._C._profiler import _EventType, _ExperimentalConfig, _ProfilerEvent
12
from torch.autograd.profiler import FunctionEvent
@@ -21,6 +20,12 @@ from vllm.profiler.utils import (
20
event_torch_op_stack_trace,
21
indent_string,
22
)
23
+from vllm.utils.import_utils import PlaceholderModule
24
+
25
+try:
26
+ import pandas as pd
27
+except ImportError:
28
+ pd = PlaceholderModule("pandas")
29
30
31
@dataclass