A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 06:30 UTC
[Benchmark] Share data between SLA runs (#32184)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 13, 2026 at 01:12 UTC
7c0d3c5152a1b68b05450e699abc90eb468d215d
2 files changed
+108
-25
tests/benchmarks/sweep/test_serve_sla.py
+68
-10
@@ -1,11 +1,12 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import json
4
from collections.abc import Callable
5
from pathlib import Path
6
from unittest.mock import patch
7
8
from vllm.benchmarks.sweep.param_sweep import ParameterSweepItem
8
-from vllm.benchmarks.sweep.serve_sla import solve_sla
9
+from vllm.benchmarks.sweep.serve_sla import _get_sla_run_path, solve_sla
10
from vllm.benchmarks.sweep.server import ServerProcess
11
from vllm.benchmarks.sweep.sla_sweep import (
12
SLACriterionBase,
@@ -34,7 +35,14 @@ def _set_return_value(
35
num_runs: int,
36
dry_run: bool,
37
):
37
- return var2metric(bench_comb)
38
+ iter_data = var2metric(bench_comb)
39
+
40
+ summary_path = _get_sla_run_path(iter_path, run_number=None)
41
+ summary_path.parent.mkdir(parents=True, exist_ok=True)
42
+ with summary_path.open("w") as f:
43
+ json.dump(iter_data, f, indent=4)
44
+
45
+ return iter_data
46
47
return patch("vllm.benchmarks.sweep.serve_sla.run_sla", side_effect=mock_run_sla)
48
@@ -98,6 +106,7 @@ def _var2metric_sqrt(y_intercept: float):
106
def _run_solve_sla(
107
var2metric: Callable[[ParameterSweepItem], list[dict[str, float]]],
108
criterion: SLACriterionBase,
109
+ base_path: Path,
110
min_value: int = 1,
111
max_value: int = 100,
112
):
@@ -108,7 +117,7 @@ def _run_solve_sla(
117
serve_comb=ParameterSweepItem(),
118
bench_comb=ParameterSweepItem(),
119
sla_comb=SLASweepItem({"request_throughput": criterion}),
111
- base_path=Path(""),
120
+ base_path=base_path,
121
num_runs=1,
122
dry_run=False,
123
sla_variable="request_rate",
@@ -120,10 +129,11 @@ def _run_solve_sla(
129
return result
130
131
123
-def test_solve_linear_sla_le():
132
+def test_solve_linear_sla_le(tmp_path):
133
sla_data, history = _run_solve_sla(
134
_var2metric_linear(),
135
SLALessThanOrEqualTo(target=32),
136
+ tmp_path,
137
)
138
139
assert history.get_max_passing() == 32
@@ -136,10 +146,11 @@ def test_solve_linear_sla_le():
146
}
147
148
139
-def test_solve_linear_sla_lt():
149
+def test_solve_linear_sla_lt(tmp_path):
150
sla_data, history = _run_solve_sla(
151
_var2metric_linear(),
152
SLALessThan(target=32),
153
+ tmp_path,
154
)
155
156
assert history.get_max_passing() == 31
@@ -152,10 +163,11 @@ def test_solve_linear_sla_lt():
163
}
164
165
155
-def test_solve_linear_sla_oob():
166
+def test_solve_linear_sla_oob(tmp_path):
167
sla_data, history = _run_solve_sla(
168
_var2metric_linear(),
169
SLALessThanOrEqualTo(target=32),
170
+ tmp_path,
171
min_value=64,
172
)
173
@@ -168,10 +180,11 @@ def test_solve_linear_sla_oob():
180
}
181
182
171
-def test_solve_concave_sla_le():
183
+def test_solve_concave_sla_le(tmp_path):
184
sla_data, history = _run_solve_sla(
185
_var2metric_concave(elbow_point=32),
186
SLALessThanOrEqualTo(target=24),
187
+ tmp_path,
188
)
189
190
assert history.get_max_passing() == 16
@@ -187,10 +200,11 @@ def test_solve_concave_sla_le():
200
}
201
202
190
-def test_solve_convex_sla_le():
203
+def test_solve_convex_sla_le(tmp_path):
204
sla_data, history = _run_solve_sla(
205
_var2metric_convex(elbow_point=32),
206
SLALessThanOrEqualTo(target=24),
207
+ tmp_path,
208
)
209
210
assert history.get_max_passing() == 26
@@ -206,10 +220,11 @@ def test_solve_convex_sla_le():
220
}
221
222
209
-def test_solve_quadratic_sla_le():
223
+def test_solve_quadratic_sla_le(tmp_path):
224
sla_data, history = _run_solve_sla(
225
_var2metric_quadratic(y_intercept=10),
226
SLALessThanOrEqualTo(target=50),
227
+ tmp_path,
228
)
229
230
assert history.get_max_passing() == 20
@@ -223,10 +238,11 @@ def test_solve_quadratic_sla_le():
238
}
239
240
226
-def test_solve_sqrt_sla_le():
241
+def test_solve_sqrt_sla_le(tmp_path):
242
sla_data, history = _run_solve_sla(
243
_var2metric_sqrt(y_intercept=10),
244
SLALessThanOrEqualTo(target=100),
245
+ tmp_path,
246
)
247
248
assert history.get_max_passing() == 81
@@ -238,3 +254,45 @@ def test_solve_sqrt_sla_le():
254
81: True,
255
82: False,
256
}
257
+
258
+
259
+def test_solve_reuse_history(tmp_path):
260
+ sla_data, history = _run_solve_sla(
261
+ _var2metric_linear(),
262
+ SLALessThanOrEqualTo(target=10),
263
+ tmp_path,
264
+ min_value=1,
265
+ max_value=20,
266
+ )
267
+
268
+ assert history.get_max_passing() == 10
269
+
270
+ assert {val: margin <= 0 for val, margin in history.items()} == {
271
+ 20: False,
272
+ 1: True,
273
+ 10: True,
274
+ 11: False,
275
+ }
276
+
277
+ sla_data, history = _run_solve_sla(
278
+ _var2metric_linear(),
279
+ SLALessThanOrEqualTo(target=30),
280
+ tmp_path,
281
+ min_value=21,
282
+ max_value=40,
283
+ )
284
+
285
+ assert history.get_max_passing() == 30
286
+
287
+ assert {val: margin <= 0 for val, margin in history.items()} == {
288
+ # Items from the past run
289
+ # (the margins are different because the target changed)
290
+ 20: True,
291
+ 1: True,
292
+ 10: True,
293
+ 11: True,
294
+ # Items from this run
295
+ 40: False,
296
+ 30: True,
297
+ 31: False,
298
+ }
vllm/benchmarks/sweep/serve_sla.py
+40
-15
@@ -65,6 +65,14 @@ def _get_sla_run_path(iter_path: Path, run_number: int | None):
65
return iter_path / f"run={run_number}.json"
66
67
68
+def _iter_sla_val_paths(base_path: Path, sla_variable: str):
69
+ for iter_path in base_path.glob(f"{sla_variable}=*"):
70
+ sla_value = int(iter_path.name.removeprefix(f"{sla_variable}="))
71
+ summary_path = iter_path / "summary.json"
72
+ if summary_path.exists():
73
+ yield sla_value, summary_path
74
+
75
+
76
def _sla_needs_server(
77
serve_comb: ParameterSweepItem,
78
bench_combs: ParameterSweep,
@@ -153,6 +161,25 @@ class SLAHistory(dict[int, float]):
161
)
162
163
164
+def _compute_margin(
165
+ sla_comb: SLASweepItem,
166
+ iter_data: list[dict[str, object]],
167
+):
168
+ assert iter_data, "Summary should not be empty"
169
+
170
+ iter_data_mean = {
171
+ k: sum(float(run_data[k]) for run_data in iter_data) / len(iter_data) # type: ignore
172
+ for k in sla_comb
173
+ }
174
+
175
+ sla_margins = [
176
+ criterion.print_and_compute_margin(iter_data_mean, k)
177
+ for k, criterion in sla_comb.items()
178
+ ]
179
+
180
+ return max(sla_margins)
181
+
182
+
183
def solve_sla(
184
server: ServerProcess | None,
185
bench_cmd: list[str],
@@ -170,11 +197,18 @@ def solve_sla(
197
sla_data = list[dict[str, object]]()
198
history = SLAHistory(min_value=sla_min_value, max_value=sla_max_value)
199
200
+ # Use results from previous runs
201
+ for past_sla_value, path in _iter_sla_val_paths(base_path, sla_variable):
202
+ with path.open("rb") as f:
203
+ past_iter_data = json.load(f)
204
+
205
+ history[past_sla_value] = _compute_margin(sla_comb, past_iter_data)
206
+
207
# NOTE: We don't use equality here to be more robust against noisy results
208
while history.get_max_passing() + 1 < history.get_min_failing():
175
- if len(history) == 0:
209
+ if max(history, default=sla_min_value) < sla_max_value:
210
val = sla_max_value
177
- elif len(history) == 1:
211
+ elif min(history, default=sla_max_value) > sla_min_value:
212
val = sla_min_value
213
else:
214
spl = PchipInterpolator(*history.get_xy(), extrapolate=False)
@@ -205,24 +239,15 @@ def solve_sla(
239
if iter_data is None:
240
return None
241
208
- sla_data.extend(iter_data)
209
-
210
- iter_data_mean = {
211
- k: sum(float(run_data[k]) for run_data in iter_data) / len(iter_data) # type: ignore
212
- for k in sla_comb
213
- }
214
-
215
- sla_margins = [
216
- criterion.print_and_compute_margin(iter_data_mean, k)
217
- for k, criterion in sla_comb.items()
218
- ]
219
- history[val] = margin = max(sla_margins)
220
-
242
+ margin = _compute_margin(sla_comb, iter_data)
243
if margin <= 0:
244
print(f"SLA criteria are met. ({margin=:.2f})")
245
else:
246
print(f"SLA criteria are not met. ({margin=:.2f})")
247
248
+ sla_data.extend(iter_data)
249
+ history[val] = margin
250
+
251
return sla_data, history
252
253