@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fd68cd132b

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 14:00 UTC

[Bugfix] Fixes for SLA finder (#35537)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Feb 28, 2026 at 12:20 UTC fd68cd132bbd8844d733ef9069e0d3b28f11c20e

5 files changed +65 -21

docs/benchmarking/sweeps.md

+27 -14

@@ -112,6 +112,7 @@ Example command:

112 vllm bench sweep serve_sla \

113 --serve-cmd 'vllm serve meta-llama/Llama-2-7b-chat-hf' \

114 --bench-cmd 'vllm bench serve --model meta-llama/Llama-2-7b-chat-hf --backend vllm --endpoint /v1/completions --dataset-name sharegpt --dataset-path benchmarks/ShareGPT_V3_unfiltered_cleaned_split.json --num-prompts 100' \

115 + --sla-variable max_concurrency \

116 --serve-params benchmarks/serve_hparams.json \

117 --bench-params benchmarks/bench_hparams.json

118 -o benchmarks/results

@@ -119,8 +120,8 @@ vllm bench sweep serve_sla \

120

121 The algorithm for scanning through different values of `sla_variable` can be summarized as follows:

122

122 -1. Run the benchmark once with `sla_variable = 1` to simulate serial inference. This results in the lowest possible latency and throughput.

123 -2. Run the benchmark once with `sla_variable = num_prompts` to simulate batch inference over the whole dataset. This results in the highest possible latency and throughput.

123 +1. Run the benchmark by sending requests one at a time (serial inference). This results in the lowest possible latency and throughput.

124 +2. Run the benchmark by sending all requests at once (batch inference). This results in the highest possible latency and throughput.

125 3. Estimate the maximum value of `sla_variable` that can be supported by the server without oversaturating it.

126 4. Run the benchmark over intermediate values of `sla_variable` uniformly using the remaining iterations.

127

@@ -129,6 +130,9 @@ You can override the number of iterations in the algorithm by setting `--sla-ite

130 !!! tip

131 This is our equivalent of [GuideLLM's `--profile sweep`](https://github.com/vllm-project/guidellm/blob/v0.5.3/src/guidellm/benchmark/profiles.py#L575).

132

133 + In general, `--sla-variable max_concurrency` produces more reliable results because it directly controls the workload imposed on the vLLM engine.

134 + Nevertheless, we default to `--sla-variable request_rate` to maintain similar behavior as GuideLLM.

135 +

136 ## Startup Benchmark

137

138 `vllm bench sweep startup` runs `vllm bench startup` across parameter combinations to compare cold/warm startup time for different engine settings.

@@ -197,23 +201,32 @@ Control the variables to plot via `--var-x` and `--var-y`, optionally applying `

201 Example commands for visualizing [SLA Scanner](#sla-scanner) results:

202

203 ```bash

200 -# Latency increases as the request rate increases

201 -vllm bench sweep plot benchmarks/results/<timestamp> \

202 - --var-x request_rate \

203 - --var-y p99_ttft_ms \

204 - --row-by random_input_len \

205 - --col-by random_output_len \

204 +# Name of the directory that stores the results

205 +TIMESTAMP=$1

206 +

207 +# Latency increases as the workload increases

208 +vllm bench sweep plot benchmarks/results/$TIMESTAMP \

209 + --var-x max_concurrency \

210 + --var-y median_ttft_ms \

211 + --col-by _benchmark_name \

212 + --curve-by max_num_seqs,max_num_batched_tokens \

213 + --fig-name latency_curve

214 +

215 +# Throughput saturates as workload increases

216 +vllm bench sweep plot benchmarks/results/$TIMESTAMP \

217 + --var-x max_concurrency \

218 + --var-y total_token_throughput \

219 + --col-by _benchmark_name \

220 --curve-by max_num_seqs,max_num_batched_tokens \

207 - --filter-by 'request_rate<=128'

221 + --fig-name throughput_curve

222

223 # Tradeoff between latency and throughput

210 -vllm bench sweep plot benchmarks/results/<timestamp> \

211 - --var-x request_throughput \

224 +vllm bench sweep plot benchmarks/results/$TIMESTAMP \

225 + --var-x total_token_throughput \

226 --var-y median_ttft_ms \

213 - --row-by random_input_len \

214 - --col-by random_output_len \

227 + --col-by _benchmark_name \

228 --curve-by max_num_seqs,max_num_batched_tokens \

216 - --filter-by 'request_rate<=128'

229 + --fig-name latency_throughput

230 ```

231

232 !!! tip

vllm/benchmarks/datasets.py

+3 -1

@@ -60,6 +60,8 @@ except ImportError:

61 logger = logging.getLogger(__name__)

63 +DEFAULT_NUM_PROMPTS = 1000

64 +

65 # -----------------------------------------------------------------------------

66 # Data Classes

67 # -----------------------------------------------------------------------------

@@ -1338,7 +1340,7 @@ def add_dataset_parser(parser: FlexibleArgumentParser):

1340 parser.add_argument(

1341 "--num-prompts",

1342 type=int,

1341 - default=1000,

1343 + default=DEFAULT_NUM_PROMPTS,

1344 help="Number of prompts to process.",

1345 )

1346 parser.add_argument(

vllm/benchmarks/sweep/plot.py

+7 -2

@@ -324,6 +324,11 @@ def _plot_fig(

324 df = filter_by.apply(df)

325 df = bin_by.apply(df)

326

327 + if len(df) == 0:

328 + print(f"No data to plot. Filters: {filter_by}")

329 + print("[END FIGURE]")

330 + return

331 +

332 # Sort by curve_by columns alphabetically for consistent legend ordering

333 if curve_by:

334 df = df.sort_values(by=curve_by)

@@ -570,13 +575,13 @@ class SweepPlotArgs:

575 parser.add_argument(

576 "--var-x",

577 type=str,

573 - default="request_throughput",

578 + default="total_token_throughput",

579 help="The variable for the x-axis.",

580 )

581 parser.add_argument(

582 "--var-y",

583 type=str,

579 - default="p99_ttft_ms",

584 + default="median_ttft_ms",

585 help="The variable for the y-axis",

586 )

587 parser.add_argument(

vllm/benchmarks/sweep/serve.py

@@ -138,12 +138,16 @@ def _get_comb_base_path(

138 output_dir: Path,

139 serve_comb: ParameterSweepItem,

140 bench_comb: ParameterSweepItem,

141 + *,

142 + extra_parts: tuple[str, ...] = (),

143 ):

144 parts = list[str]()

145 if serve_comb:

146 parts.extend(("SERVE-", serve_comb.name))

147 if bench_comb:

148 parts.extend(("BENCH-", bench_comb.name))

149 + if extra_parts:

150 + parts.extend(extra_parts)

151

152 return output_dir / sanitize_filename("-".join(parts))

153

vllm/benchmarks/sweep/serve_sla.py

+24 -4

@@ -10,6 +10,7 @@ from typing import ClassVar, Literal, get_args

10 import numpy as np

11 from typing_extensions import assert_never

13 +from vllm.benchmarks.datasets import DEFAULT_NUM_PROMPTS

14 from vllm.utils.import_utils import PlaceholderModule

16 from .param_sweep import ParameterSweep, ParameterSweepItem

@@ -65,7 +66,12 @@ def run_comb_sla(

66 bench_cmd,

67 serve_comb=serve_comb,

68 bench_comb=bench_comb_sla,

68 - base_path=_get_comb_base_path(output_dir, serve_comb, bench_comb_sla),

69 + base_path=_get_comb_base_path(

70 + output_dir,

71 + serve_comb,

72 + bench_comb,

73 + extra_parts=("SLA-", f"{sla_variable}={sla_value}"),

74 + ),

75 num_runs=num_runs,

76 dry_run=dry_run,

77 link_vars=link_vars,

@@ -93,11 +99,25 @@ def explore_sla(

99 if sla_iters < 2:

100 raise ValueError("`sla_iters` should be at least 2")

101

102 + dataset_size = DEFAULT_NUM_PROMPTS

103 + if "num_prompts" in bench_comb:

104 + dataset_size = int(bench_comb["num_prompts"]) # type: ignore

105 + else:

106 + for i, arg in enumerate(bench_cmd):

107 + if arg == "--num-prompts" and i + 1 < len(bench_cmd):

108 + dataset_size = int(bench_cmd[i + 1])

109 + break

110 + elif arg.startswith("--num-prompts="):

111 + dataset_size = int(arg.split("=", 1)[1])

112 + break

113 +

114 + print(f"Dataset size: {dataset_size}")

115 +

116 serial_comb_data = run_comb_sla(

117 server,

118 bench_cmd,

119 serve_comb=serve_comb,

100 - bench_comb=bench_comb,

120 + bench_comb=bench_comb | {"max_concurrency": 1},

121 output_dir=output_dir,

122 num_runs=num_runs,

123 dry_run=dry_run,

@@ -109,13 +129,13 @@ def explore_sla(

129 server,

130 bench_cmd,

131 serve_comb=serve_comb,

112 - bench_comb=bench_comb,

132 + bench_comb=bench_comb | {"max_concurrency": dataset_size},

133 output_dir=output_dir,

134 num_runs=num_runs,

135 dry_run=dry_run,

136 link_vars=link_vars,

137 sla_variable=sla_variable,

118 - sla_value=int(bench_comb.get("num_prompts", 1000)), # type: ignore

138 + sla_value=dataset_size,

139 )

140

141 if serial_comb_data is None or batch_comb_data is None: