A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 09:00 UTC
[Benchmark][2/2] Use spline interpolation to tune SLA variables (#32095)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 11, 2026 at 12:27 UTC
ef96fa3f1f15e08769e70ee3e335b5b4d7e6a6ee
3 files changed
+209
-241
docs/benchmarking/sweeps.md
+4
-4
@@ -129,10 +129,10 @@ vllm bench sweep serve_sla \
129
130
The algorithm for adjusting the SLA variable is as follows:
131
132
-1. Run the benchmark with infinite QPS, and use the corresponding metrics to determine the initial value of the variable.
133
- - For example, the initial request rate is set to the concurrency under infinite QPS.
134
-2. If the SLA is still satisfied, keep doubling the value until the SLA is no longer satisfied. This gives a relatively narrow window that contains the point where the SLA is barely satisfied.
135
-3. Apply binary search over the window to find the maximum value that still satisfies the SLA.
132
+1. Run the benchmark once with maximum possible QPS, and once with minimum possible QPS. For each run, calculate the distance of the SLA metrics from their targets, resulting in data points of QPS vs SLA distance.
133
+2. Perform spline interpolation between the data points to estimate the QPS that results in zero SLA distance.
134
+3. Run the benchmark with the estimated QPS and add the resulting data point to the history.
135
+4. Repeat Steps 2 and 3 until the maximum QPS that passes SLA and the minimum QPS that fails SLA in the history are close enough to each other.
136
137
!!! important
138
SLA tuning is applied over each combination of `--serve-params`, `--bench-params`, and `--sla-params`.
tests/benchmarks/sweep/test_serve_sla.py
+135
-97
@@ -5,7 +5,7 @@ from pathlib import Path
5
from unittest.mock import patch
6
7
from vllm.benchmarks.sweep.param_sweep import ParameterSweepItem
8
-from vllm.benchmarks.sweep.serve_sla import _estimate_sla_bounds, _find_sla_value
8
+from vllm.benchmarks.sweep.serve_sla import solve_sla
9
from vllm.benchmarks.sweep.server import ServerProcess
10
from vllm.benchmarks.sweep.sla_sweep import (
11
SLACriterionBase,
@@ -39,18 +39,70 @@ def _set_return_value(
39
return patch("vllm.benchmarks.sweep.serve_sla.run_sla", side_effect=mock_run_sla)
40
41
42
-def _var2metric_identity(bench_comb):
43
- return [{"request_throughput": float(bench_comb["request_rate"])}]
42
+def _var2metric_linear():
43
+ def wrapped(bench_comb):
44
+ x = float(bench_comb["request_rate"])
45
+ y = x
46
47
+ return [{"request_throughput": y}]
48
46
-def _run_estimate_sla_bounds(
49
+ return wrapped
50
+
51
+
52
+def _var2metric_concave(elbow_point: float):
53
+ def wrapped(bench_comb):
54
+ x = float(bench_comb["request_rate"])
55
+ if x < elbow_point:
56
+ y = 0.5 * (x - elbow_point) + elbow_point
57
+ else:
58
+ y = 1.5 * (x - elbow_point) + elbow_point
59
+
60
+ return [{"request_throughput": y}]
61
+
62
+ return wrapped
63
+
64
+
65
+def _var2metric_convex(elbow_point: float):
66
+ def wrapped(bench_comb):
67
+ x = float(bench_comb["request_rate"])
68
+ if x < elbow_point:
69
+ y = 1.5 * (x - elbow_point) + elbow_point
70
+ else:
71
+ y = 0.5 * (x - elbow_point) + elbow_point
72
+
73
+ return [{"request_throughput": y}]
74
+
75
+ return wrapped
76
+
77
+
78
+def _var2metric_quadratic(y_intercept: float):
79
+ def wrapped(bench_comb):
80
+ x = float(bench_comb["request_rate"])
81
+ y = y_intercept + 0.1 * x**2
82
+
83
+ return [{"request_throughput": y}]
84
+
85
+ return wrapped
86
+
87
+
88
+def _var2metric_sqrt(y_intercept: float):
89
+ def wrapped(bench_comb):
90
+ x = float(bench_comb["request_rate"])
91
+ y = y_intercept + 10 * x**0.5
92
+
93
+ return [{"request_throughput": y}]
94
+
95
+ return wrapped
96
+
97
+
98
+def _run_solve_sla(
99
var2metric: Callable[[ParameterSweepItem], list[dict[str, float]]],
100
criterion: SLACriterionBase,
49
- init_value: int,
50
- max_value: int,
101
+ min_value: int = 1,
102
+ max_value: int = 100,
103
):
104
with _set_return_value(var2metric):
53
- return _estimate_sla_bounds(
105
+ result = solve_sla(
106
server=None,
107
bench_cmd=[],
108
serve_comb=ParameterSweepItem(),
@@ -60,143 +112,129 @@ def _run_estimate_sla_bounds(
112
num_runs=1,
113
dry_run=False,
114
sla_variable="request_rate",
63
- init_value=init_value,
64
- max_value=max_value,
115
+ sla_min_value=min_value,
116
+ sla_max_value=max_value,
117
)
118
+ assert result is not None
119
120
+ return result
121
68
-def test_estimate_sla_bounds_le():
69
- sla_data, (max_passing, min_failing), history = _run_estimate_sla_bounds(
70
- _var2metric_identity,
122
+
123
+def test_solve_linear_sla_le():
124
+ sla_data, history = _run_solve_sla(
125
+ _var2metric_linear(),
126
SLALessThanOrEqualTo(target=32),
72
- init_value=1,
73
- max_value=100,
127
)
128
76
- assert max_passing == 32
77
- assert min_failing == 64
129
+ assert history.get_max_passing() == 32
130
131
assert {val: margin <= 0 for val, margin in history.items()} == {
132
+ 100: False,
133
1: True,
81
- 2: True,
82
- 4: True,
83
- 8: True,
84
- 16: True,
134
32: True,
86
- 64: False,
135
+ 33: False,
136
}
137
138
90
-def test_estimate_sla_bounds_lt():
91
- sla_data, (max_passing, min_failing), history = _run_estimate_sla_bounds(
92
- _var2metric_identity,
139
+def test_solve_linear_sla_lt():
140
+ sla_data, history = _run_solve_sla(
141
+ _var2metric_linear(),
142
SLALessThan(target=32),
94
- init_value=1,
95
- max_value=100,
143
)
144
98
- assert max_passing == 16
99
- assert min_failing == 32
145
+ assert history.get_max_passing() == 31
146
147
assert {val: margin <= 0 for val, margin in history.items()} == {
148
+ 100: False,
149
1: True,
103
- 2: True,
104
- 4: True,
105
- 8: True,
106
- 16: True,
150
+ 31: True,
151
32: False,
152
}
153
154
111
-def test_estimate_sla_bounds_oob():
112
- sla_data, (max_passing, min_failing), history = _run_estimate_sla_bounds(
113
- _var2metric_identity,
155
+def test_solve_linear_sla_oob():
156
+ sla_data, history = _run_solve_sla(
157
+ _var2metric_linear(),
158
SLALessThanOrEqualTo(target=32),
115
- init_value=64,
116
- max_value=128,
159
+ min_value=64,
160
)
161
119
- assert max_passing == 0
120
- assert min_failing == 64
162
+ assert history.get_max_passing() == 64
163
+ assert history.get_min_failing() == 64
164
165
assert {val: margin <= 0 for val, margin in history.items()} == {
166
+ 100: False,
167
64: False,
168
}
169
170
127
-def _run_test_find_sla_value_le(
128
- var2metric: Callable[[ParameterSweepItem], list[dict[str, float]]],
129
- criterion: SLACriterionBase,
130
- min_value: int,
131
- max_value: int,
132
-):
133
- with _set_return_value(var2metric):
134
- return _find_sla_value(
135
- server=None,
136
- bench_cmd=[],
137
- serve_comb=ParameterSweepItem(),
138
- bench_comb=ParameterSweepItem(),
139
- sla_comb=SLASweepItem({"request_throughput": criterion}),
140
- base_path=Path(""),
141
- num_runs=1,
142
- dry_run=False,
143
- sla_variable="request_rate",
144
- min_value=min_value,
145
- max_value=max_value,
146
- )
171
+def test_solve_concave_sla_le():
172
+ sla_data, history = _run_solve_sla(
173
+ _var2metric_concave(elbow_point=32),
174
+ SLALessThanOrEqualTo(target=24),
175
+ )
176
177
+ assert history.get_max_passing() == 16
178
149
-def test_find_sla_value_le():
150
- sla_data, sla_value, history = _run_test_find_sla_value_le(
151
- _var2metric_identity,
152
- SLALessThanOrEqualTo(target=50.0),
153
- min_value=32,
154
- max_value=64,
179
+ assert {val: margin <= 0 for val, margin in history.items()} == {
180
+ 100: False,
181
+ 1: True,
182
+ 7: True,
183
+ 13: True,
184
+ 15: True,
185
+ 16: True,
186
+ 17: False,
187
+ }
188
+
189
+
190
+def test_solve_convex_sla_le():
191
+ sla_data, history = _run_solve_sla(
192
+ _var2metric_convex(elbow_point=32),
193
+ SLALessThanOrEqualTo(target=24),
194
)
195
157
- assert sla_value == 50
196
+ assert history.get_max_passing() == 26
197
+
198
assert {val: margin <= 0 for val, margin in history.items()} == {
159
- 48: True,
160
- 56: False,
161
- 52: False,
162
- 50: True,
163
- 51: False,
199
+ 100: False,
200
+ 1: True,
201
+ 48: False,
202
+ 30: False,
203
+ 24: True,
204
+ 26: True,
205
+ 27: False,
206
}
207
208
167
-def test_find_sla_value_lt():
168
- sla_data, sla_value, history = _run_test_find_sla_value_le(
169
- _var2metric_identity,
170
- SLALessThan(target=50.0),
171
- min_value=32,
172
- max_value=64,
209
+def test_solve_quadratic_sla_le():
210
+ sla_data, history = _run_solve_sla(
211
+ _var2metric_quadratic(y_intercept=10),
212
+ SLALessThanOrEqualTo(target=50),
213
)
214
175
- assert sla_value == 49
215
+ assert history.get_max_passing() == 20
216
+
217
assert {val: margin <= 0 for val, margin in history.items()} == {
177
- 48: True,
178
- 56: False,
179
- 52: False,
180
- 50: False,
181
- 49: True,
218
+ 100: False,
219
+ 1: True,
220
+ 4: True,
221
+ 20: True,
222
+ 21: False,
223
}
224
225
185
-def test_find_sla_value_oob():
186
- sla_data, sla_value, history = _run_test_find_sla_value_le(
187
- _var2metric_identity,
188
- SLALessThanOrEqualTo(target=50.0),
189
- min_value=64,
190
- max_value=128,
226
+def test_solve_sqrt_sla_le():
227
+ sla_data, history = _run_solve_sla(
228
+ _var2metric_sqrt(y_intercept=10),
229
+ SLALessThanOrEqualTo(target=100),
230
)
231
193
- assert sla_value == 64
232
+ assert history.get_max_passing() == 81
233
+
234
assert {val: margin <= 0 for val, margin in history.items()} == {
195
- 96: False,
196
- 80: False,
197
- 72: False,
198
- 68: False,
199
- 66: False,
200
- 65: False,
201
- 64: False,
235
+ 100: False,
236
+ 1: True,
237
+ 89: False,
238
+ 81: True,
239
+ 82: False,
240
}
vllm/benchmarks/sweep/serve_sla.py
+70
-140
@@ -3,14 +3,11 @@
3
import argparse
4
import contextlib
5
import json
6
-import math
6
from dataclasses import asdict, dataclass
7
from datetime import datetime
8
from pathlib import Path
9
from typing import ClassVar, Literal, get_args
10
12
-from typing_extensions import assert_never
13
-
11
from vllm.utils.import_utils import PlaceholderModule
12
13
from .param_sweep import ParameterSweep, ParameterSweepItem
@@ -24,6 +21,15 @@ try:
21
except ImportError:
22
pd = PlaceholderModule("pandas")
23
24
+try:
25
+ from scipy.interpolate import PchipInterpolator
26
+except ImportError:
27
+ PchipInterpolator = (
28
+ PlaceholderModule("scipy")
29
+ .placeholder_attr("interpolate")
30
+ .placeholder_attr("PchipInterpolator")
31
+ )
32
+
33
34
def _get_sla_base_path(
35
output_dir: Path,
@@ -118,89 +124,36 @@ def run_sla(
124
SLAVariable = Literal["request_rate", "max_concurrency"]
125
126
121
-def _estimate_sla_value(run_data: dict[str, object], sla_variable: SLAVariable):
122
- request_throughput = float(run_data["request_throughput"]) # type: ignore
123
- if sla_variable == "request_rate":
124
- return request_throughput
125
- if sla_variable == "max_concurrency":
126
- mean_latency_ms = float(run_data["mean_e2el_ms"]) # type: ignore
127
- return request_throughput * mean_latency_ms / 1000
127
+class SLAHistory(dict[int, float]):
128
+ def __init__(self, min_value: int, max_value: int) -> None:
129
+ super().__init__()
130
129
- assert_never(sla_variable)
131
+ self.min_value = min_value
132
+ self.max_value = max_value
133
134
+ def get_xy(self) -> tuple[list[int], list[float]]:
135
+ xs = list[int]()
136
+ ys = list[float]()
137
+ for x, y in sorted(self.items()):
138
+ xs.append(x)
139
+ ys.append(y)
140
132
-def _estimate_sla_bounds(
133
- server: ServerProcess | None,
134
- bench_cmd: list[str],
135
- *,
136
- serve_comb: ParameterSweepItem,
137
- bench_comb: ParameterSweepItem,
138
- sla_comb: SLASweepItem,
139
- base_path: Path,
140
- num_runs: int,
141
- dry_run: bool,
142
- sla_variable: SLAVariable,
143
- init_value: int,
144
- max_value: int,
145
-):
146
- sla_data = list[dict[str, object]]()
147
-
148
- val: int = init_value
149
- assert val > 0
150
-
151
- history = dict[int, float]()
152
-
153
- while True:
154
- print(f"Testing {sla_variable}: {val} req/s")
141
+ return xs, ys
142
156
- iter_data = run_sla(
157
- server,
158
- bench_cmd,
159
- serve_comb=serve_comb,
160
- bench_comb=bench_comb | {sla_variable: val},
161
- iter_path=_get_sla_iter_path(base_path, sla_comb, sla_variable, val),
162
- num_runs=num_runs,
163
- dry_run=dry_run,
143
+ def get_max_passing(self) -> float:
144
+ return max(
145
+ (val for val, margin in self.items() if margin <= 0),
146
+ default=self.min_value,
147
)
148
166
- assert iter_data is not None
167
- sla_data.extend(iter_data)
168
-
169
- iter_data_mean = {
170
- k: sum(float(run_data[k]) for run_data in iter_data) / len(iter_data) # type: ignore
171
- for k in sla_comb
172
- }
173
-
174
- sla_margins = [
175
- criterion.print_and_compute_margin(iter_data_mean, k)
176
- for k, criterion in sla_comb.items()
177
- ]
178
- margin = max(sla_margins)
179
- history[val] = margin
180
-
181
- if margin <= 0:
182
- print("SLA criteria are met.")
183
- val *= 2
184
- else:
185
- print("SLA criteria are not met.")
186
- break
187
-
188
- if val >= max_value:
189
- break
190
-
191
- max_passing = max(
192
- (val for val, margin in history.items() if margin <= 0),
193
- default=0,
194
- )
195
- min_failing = min(
196
- (val for val, margin in history.items() if margin > 0),
197
- default=max_value,
198
- )
199
-
200
- return sla_data, (max_passing, min_failing), history
149
+ def get_min_failing(self) -> float:
150
+ return min(
151
+ (val for val, margin in self.items() if margin > 0),
152
+ default=self.max_value,
153
+ )
154
155
203
-def _find_sla_value(
156
+def solve_sla(
157
server: ServerProcess | None,
158
bench_cmd: list[str],
159
*,
@@ -211,18 +164,33 @@ def _find_sla_value(
164
num_runs: int,
165
dry_run: bool,
166
sla_variable: SLAVariable,
214
- min_value: int,
215
- max_value: int,
167
+ sla_min_value: int = 1,
168
+ sla_max_value: int = 8192, # The value that represents infinite QPS
169
):
170
sla_data = list[dict[str, object]]()
218
-
219
- left: int = min_value
220
- right: int = max_value
221
-
222
- history = dict[int, float]()
223
-
224
- while True:
225
- val = (left + right) // 2
171
+ history = SLAHistory(min_value=sla_min_value, max_value=sla_max_value)
172
+
173
+ # NOTE: We don't use equality here to be more robust against noisy results
174
+ while history.get_max_passing() + 1 < history.get_min_failing():
175
+ if len(history) == 0:
176
+ val = sla_max_value
177
+ elif len(history) == 1:
178
+ val = sla_min_value
179
+ else:
180
+ spl = PchipInterpolator(*history.get_xy(), extrapolate=False)
181
+ spl_roots = spl.solve()
182
+ if len(spl_roots) == 0:
183
+ # Fallback to binary search
184
+ val = int((history.get_max_passing() + history.get_min_failing()) / 2)
185
+ else:
186
+ val = int(spl_roots[0])
187
+
188
+ if val in history:
189
+ # Cover both sides (floor and ceil) of the root to be sure
190
+ # that it is indeed the target value
191
+ val += 1
192
+
193
+ val = max(sla_min_value, min(val, sla_max_value))
194
print(f"Testing {sla_variable}: {val} req/s")
195
196
iter_data = run_sla(
@@ -234,8 +202,9 @@ def _find_sla_value(
202
num_runs=num_runs,
203
dry_run=dry_run,
204
)
205
+ if iter_data is None:
206
+ return None
207
238
- assert iter_data is not None
208
sla_data.extend(iter_data)
209
210
iter_data_mean = {
@@ -247,20 +216,14 @@ def _find_sla_value(
216
criterion.print_and_compute_margin(iter_data_mean, k)
217
for k, criterion in sla_comb.items()
218
]
250
- margin = max(sla_margins)
251
- history[val] = margin
219
+ history[val] = margin = max(sla_margins)
220
221
if margin <= 0:
254
- print("SLA criteria are met.")
255
- left = val
222
+ print(f"SLA criteria are met. ({margin=:.2f})")
223
else:
257
- print("SLA criteria are not met.")
258
- right = val
259
-
260
- if right - left <= 1 and left in history:
261
- break
224
+ print(f"SLA criteria are not met. ({margin=:.2f})")
225
263
- return sla_data, left, history
226
+ return sla_data, history
227
228
229
def search_sla(
@@ -271,7 +234,6 @@ def search_sla(
234
bench_comb: ParameterSweepItem,
235
sla_comb: SLASweepItem,
236
sla_variable: SLAVariable,
274
- sla_inf_value: int = 65536, # The value that represents infinite QPS
237
base_path: Path,
238
num_runs: int,
239
dry_run: bool,
@@ -279,43 +241,7 @@ def search_sla(
241
print("[SLA START]")
242
print(f"SLA criteria: {sla_comb.as_text()}")
243
282
- sla_data_0 = run_sla(
283
- server,
284
- bench_cmd,
285
- serve_comb=serve_comb,
286
- bench_comb=bench_comb | {sla_variable: sla_inf_value},
287
- iter_path=_get_sla_iter_path(base_path, sla_comb, sla_variable, sla_inf_value),
288
- num_runs=num_runs,
289
- dry_run=dry_run,
290
- )
291
- if sla_data_0 is None:
292
- assert dry_run
293
- print("Omitting SLA search.")
294
- print("[SLA END]")
295
- return None
296
-
297
- sla_init_value = math.ceil(
298
- sum(_estimate_sla_value(item, sla_variable) for item in sla_data_0)
299
- / len(sla_data_0)
300
- )
301
- print(f"Initial {sla_variable} to search: {sla_init_value} req/s.")
302
-
303
- sla_data_1, (sla_min, sla_max), _ = _estimate_sla_bounds(
304
- server,
305
- bench_cmd,
306
- serve_comb=serve_comb,
307
- bench_comb=bench_comb,
308
- sla_comb=sla_comb,
309
- base_path=base_path,
310
- num_runs=num_runs,
311
- dry_run=dry_run,
312
- sla_variable=sla_variable,
313
- init_value=sla_init_value,
314
- max_value=sla_inf_value,
315
- )
316
- print(f"Range of {sla_variable} to search: [{sla_min}, {sla_max}] req/s.")
317
-
318
- sla_data_2, sla_value, _ = _find_sla_value(
244
+ result = solve_sla(
245
server,
246
bench_cmd,
247
serve_comb=serve_comb,
@@ -325,11 +251,15 @@ def search_sla(
251
num_runs=num_runs,
252
dry_run=dry_run,
253
sla_variable=sla_variable,
328
- min_value=sla_min,
329
- max_value=sla_max,
254
)
255
+ if result is None:
256
+ assert dry_run
257
+ print("Omitting SLA search.")
258
+ print("[SLA END]")
259
+ return
260
332
- sla_data = sla_data_0 + sla_data_1 + sla_data_2
261
+ sla_data, sla_history = result
262
+ sla_value = sla_history.get_max_passing()
263
print(f"Maximum {sla_variable} for SLA: {sla_value} req/s.")
264
265
with _get_sla_iter_path(