A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 09:00 UTC
[Benchmark][1/2] Generalize SLA criterion validation from binary flags to margins (#32075)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 10, 2026 at 15:11 UTC
5f2385a4c8daac7e15142ebf96bc9a91acd61283
6 files changed
+253
-32
tests/benchmarks/sweep/__init__.py
tests/benchmarks/sweep/test_param_sweep.py
renamed
tests/benchmarks/sweep/test_serve_sla.py
new
+202
@@ -0,0 +1,202 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from collections.abc import Callable
4
+from pathlib import Path
5
+from unittest.mock import patch
6
+
7
+from vllm.benchmarks.sweep.param_sweep import ParameterSweepItem
8
+from vllm.benchmarks.sweep.serve_sla import _estimate_sla_bounds, _find_sla_value
9
+from vllm.benchmarks.sweep.server import ServerProcess
10
+from vllm.benchmarks.sweep.sla_sweep import (
11
+ SLACriterionBase,
12
+ SLALessThan,
13
+ SLALessThanOrEqualTo,
14
+ SLASweepItem,
15
+)
16
+
17
+
18
+def _set_return_value(
19
+ var2metric: Callable[[ParameterSweepItem], list[dict[str, float]]],
20
+):
21
+ """
22
+ Create a patch for run_sla with a specific function
23
+ indicating the relationship between the benchmark combination
24
+ (which includes the SLA variable) and the SLA criterion.
25
+ """
26
+
27
+ def mock_run_sla(
28
+ server: ServerProcess | None,
29
+ bench_cmd: list[str],
30
+ *,
31
+ serve_comb: ParameterSweepItem,
32
+ bench_comb: ParameterSweepItem,
33
+ iter_path: Path,
34
+ num_runs: int,
35
+ dry_run: bool,
36
+ ):
37
+ return var2metric(bench_comb)
38
+
39
+ return patch("vllm.benchmarks.sweep.serve_sla.run_sla", side_effect=mock_run_sla)
40
+
41
+
42
+def _var2metric_identity(bench_comb):
43
+ return [{"request_throughput": float(bench_comb["request_rate"])}]
44
+
45
+
46
+def _run_estimate_sla_bounds(
47
+ var2metric: Callable[[ParameterSweepItem], list[dict[str, float]]],
48
+ criterion: SLACriterionBase,
49
+ init_value: int,
50
+ max_value: int,
51
+):
52
+ with _set_return_value(var2metric):
53
+ return _estimate_sla_bounds(
54
+ server=None,
55
+ bench_cmd=[],
56
+ serve_comb=ParameterSweepItem(),
57
+ bench_comb=ParameterSweepItem(),
58
+ sla_comb=SLASweepItem({"request_throughput": criterion}),
59
+ base_path=Path(""),
60
+ num_runs=1,
61
+ dry_run=False,
62
+ sla_variable="request_rate",
63
+ init_value=init_value,
64
+ max_value=max_value,
65
+ )
66
+
67
+
68
+def test_estimate_sla_bounds_le():
69
+ sla_data, (max_passing, min_failing), history = _run_estimate_sla_bounds(
70
+ _var2metric_identity,
71
+ SLALessThanOrEqualTo(target=32),
72
+ init_value=1,
73
+ max_value=100,
74
+ )
75
+
76
+ assert max_passing == 32
77
+ assert min_failing == 64
78
+
79
+ assert {val: margin <= 0 for val, margin in history.items()} == {
80
+ 1: True,
81
+ 2: True,
82
+ 4: True,
83
+ 8: True,
84
+ 16: True,
85
+ 32: True,
86
+ 64: False,
87
+ }
88
+
89
+
90
+def test_estimate_sla_bounds_lt():
91
+ sla_data, (max_passing, min_failing), history = _run_estimate_sla_bounds(
92
+ _var2metric_identity,
93
+ SLALessThan(target=32),
94
+ init_value=1,
95
+ max_value=100,
96
+ )
97
+
98
+ assert max_passing == 16
99
+ assert min_failing == 32
100
+
101
+ assert {val: margin <= 0 for val, margin in history.items()} == {
102
+ 1: True,
103
+ 2: True,
104
+ 4: True,
105
+ 8: True,
106
+ 16: True,
107
+ 32: False,
108
+ }
109
+
110
+
111
+def test_estimate_sla_bounds_oob():
112
+ sla_data, (max_passing, min_failing), history = _run_estimate_sla_bounds(
113
+ _var2metric_identity,
114
+ SLALessThanOrEqualTo(target=32),
115
+ init_value=64,
116
+ max_value=128,
117
+ )
118
+
119
+ assert max_passing == 0
120
+ assert min_failing == 64
121
+
122
+ assert {val: margin <= 0 for val, margin in history.items()} == {
123
+ 64: False,
124
+ }
125
+
126
+
127
+def _run_test_find_sla_value_le(
128
+ var2metric: Callable[[ParameterSweepItem], list[dict[str, float]]],
129
+ criterion: SLACriterionBase,
130
+ min_value: int,
131
+ max_value: int,
132
+):
133
+ with _set_return_value(var2metric):
134
+ return _find_sla_value(
135
+ server=None,
136
+ bench_cmd=[],
137
+ serve_comb=ParameterSweepItem(),
138
+ bench_comb=ParameterSweepItem(),
139
+ sla_comb=SLASweepItem({"request_throughput": criterion}),
140
+ base_path=Path(""),
141
+ num_runs=1,
142
+ dry_run=False,
143
+ sla_variable="request_rate",
144
+ min_value=min_value,
145
+ max_value=max_value,
146
+ )
147
+
148
+
149
+def test_find_sla_value_le():
150
+ sla_data, sla_value, history = _run_test_find_sla_value_le(
151
+ _var2metric_identity,
152
+ SLALessThanOrEqualTo(target=50.0),
153
+ min_value=32,
154
+ max_value=64,
155
+ )
156
+
157
+ assert sla_value == 50
158
+ assert {val: margin <= 0 for val, margin in history.items()} == {
159
+ 48: True,
160
+ 56: False,
161
+ 52: False,
162
+ 50: True,
163
+ 51: False,
164
+ }
165
+
166
+
167
+def test_find_sla_value_lt():
168
+ sla_data, sla_value, history = _run_test_find_sla_value_le(
169
+ _var2metric_identity,
170
+ SLALessThan(target=50.0),
171
+ min_value=32,
172
+ max_value=64,
173
+ )
174
+
175
+ assert sla_value == 49
176
+ assert {val: margin <= 0 for val, margin in history.items()} == {
177
+ 48: True,
178
+ 56: False,
179
+ 52: False,
180
+ 50: False,
181
+ 49: True,
182
+ }
183
+
184
+
185
+def test_find_sla_value_oob():
186
+ sla_data, sla_value, history = _run_test_find_sla_value_le(
187
+ _var2metric_identity,
188
+ SLALessThanOrEqualTo(target=50.0),
189
+ min_value=64,
190
+ max_value=128,
191
+ )
192
+
193
+ assert sla_value == 64
194
+ assert {val: margin <= 0 for val, margin in history.items()} == {
195
+ 96: False,
196
+ 80: False,
197
+ 72: False,
198
+ 68: False,
199
+ 66: False,
200
+ 65: False,
201
+ 64: False,
202
+ }
vllm/benchmarks/sweep/param_sweep.py
+2
-1
@@ -74,7 +74,8 @@ class ParameterSweepItem(dict[str, object]):
74
representation of all parameters.
75
"""
76
if "_benchmark_name" in self:
77
- return self["_benchmark_name"]
77
+ return str(self["_benchmark_name"])
78
+
79
return self.as_text(sep="-")
80
81
# In JSON, we prefer "_"
vllm/benchmarks/sweep/serve_sla.py
+28
-16
@@ -145,12 +145,11 @@ def _estimate_sla_bounds(
145
):
146
sla_data = list[dict[str, object]]()
147
148
- max_passing: int = 0
149
- min_failing: int = 0
150
-
148
val: int = init_value
149
assert val > 0
150
151
+ history = dict[int, float]()
152
+
153
while True:
154
print(f"Testing {sla_variable}: {val} req/s")
155
@@ -172,24 +171,33 @@ def _estimate_sla_bounds(
171
for k in sla_comb
172
}
173
175
- sla_results = [
176
- criterion.print_and_validate(iter_data_mean, k)
174
+ sla_margins = [
175
+ criterion.print_and_compute_margin(iter_data_mean, k)
176
for k, criterion in sla_comb.items()
177
]
178
+ margin = max(sla_margins)
179
+ history[val] = margin
180
180
- if all(sla_results):
181
+ if margin <= 0:
182
print("SLA criteria are met.")
182
- max_passing = val
183
val *= 2
184
else:
185
print("SLA criteria are not met.")
186
- min_failing = val
186
break
187
188
if val >= max_value:
189
break
190
192
- return sla_data, (max_passing, min_failing)
191
+ max_passing = max(
192
+ (val for val, margin in history.items() if margin <= 0),
193
+ default=0,
194
+ )
195
+ min_failing = min(
196
+ (val for val, margin in history.items() if margin > 0),
197
+ default=max_value,
198
+ )
199
+
200
+ return sla_data, (max_passing, min_failing), history
201
202
203
def _find_sla_value(
@@ -211,6 +219,8 @@ def _find_sla_value(
219
left: int = min_value
220
right: int = max_value
221
222
+ history = dict[int, float]()
223
+
224
while True:
225
val = (left + right) // 2
226
print(f"Testing {sla_variable}: {val} req/s")
@@ -233,22 +243,24 @@ def _find_sla_value(
243
for k in sla_comb
244
}
245
236
- sla_results = [
237
- criterion.print_and_validate(iter_data_mean, k)
246
+ sla_margins = [
247
+ criterion.print_and_compute_margin(iter_data_mean, k)
248
for k, criterion in sla_comb.items()
249
]
250
+ margin = max(sla_margins)
251
+ history[val] = margin
252
241
- if all(sla_results):
253
+ if margin <= 0:
254
print("SLA criteria are met.")
255
left = val
256
else:
257
print("SLA criteria are not met.")
258
right = val
259
248
- if right - left <= 1:
260
+ if right - left <= 1 and left in history:
261
break
262
251
- return sla_data, left
263
+ return sla_data, left, history
264
265
266
def search_sla(
@@ -288,7 +300,7 @@ def search_sla(
300
)
301
print(f"Initial {sla_variable} to search: {sla_init_value} req/s.")
302
291
- sla_data_1, (sla_min, sla_max) = _estimate_sla_bounds(
303
+ sla_data_1, (sla_min, sla_max), _ = _estimate_sla_bounds(
304
server,
305
bench_cmd,
306
serve_comb=serve_comb,
@@ -303,7 +315,7 @@ def search_sla(
315
)
316
print(f"Range of {sla_variable} to search: [{sla_min}, {sla_max}] req/s.")
317
306
- sla_data_2, sla_value = _find_sla_value(
318
+ sla_data_2, sla_value, _ = _find_sla_value(
319
server,
320
bench_cmd,
321
serve_comb=serve_comb,
vllm/benchmarks/sweep/sla_sweep.py
+21
-15
@@ -7,39 +7,45 @@ from dataclasses import dataclass
7
8
from typing_extensions import override
9
10
+SLA_EPS = 1e-8
11
+"""Offset used to differentiate margins for equality checks."""
12
+
13
14
@dataclass
15
class SLACriterionBase(ABC):
16
target: float
17
18
@abstractmethod
16
- def validate(self, actual: float) -> bool:
17
- """Return `True` if this criterion is met; otherwise `False`."""
19
+ def compute_margin(self, actual: float) -> float:
20
+ """
21
+ Return a negative value or `0` if this criterion is met;
22
+ otherwise a positive value indicating the distance to the target.
23
+ """
24
raise NotImplementedError
25
26
@abstractmethod
27
def format_cond(self, lhs: str) -> str:
28
raise NotImplementedError
29
24
- def print_and_validate(
30
+ def print_and_compute_margin(
31
self,
32
metrics: dict[str, float],
33
metrics_key: str,
28
- ) -> bool:
34
+ ) -> float:
35
metric = metrics[metrics_key]
30
- result = self.validate(metric)
36
+ margin = self.compute_margin(metric)
37
38
cond = self.format_cond(f"{metrics_key} = {metric:.2f}")
33
- print(f"Validating SLA: {cond} | " + ("PASSED" if result else "FAILED"))
39
+ print(f"Validating SLA: {cond} | " + ("PASSED" if margin <= 0 else "FAILED"))
40
35
- return result
41
+ return margin
42
43
44
@dataclass
45
class SLALessThan(SLACriterionBase):
46
@override
41
- def validate(self, actual: float) -> bool:
42
- return actual < self.target
47
+ def compute_margin(self, actual: float) -> float:
48
+ return actual + SLA_EPS - self.target
49
50
@override
51
def format_cond(self, lhs: str) -> str:
@@ -49,8 +55,8 @@ class SLALessThan(SLACriterionBase):
55
@dataclass
56
class SLALessThanOrEqualTo(SLACriterionBase):
57
@override
52
- def validate(self, actual: float) -> bool:
53
- return actual <= self.target
58
+ def compute_margin(self, actual: float) -> float:
59
+ return actual - self.target
60
61
@override
62
def format_cond(self, lhs: str) -> str:
@@ -60,8 +66,8 @@ class SLALessThanOrEqualTo(SLACriterionBase):
66
@dataclass
67
class SLAGreaterThan(SLACriterionBase):
68
@override
63
- def validate(self, actual: float) -> bool:
64
- return actual > self.target
69
+ def compute_margin(self, actual: float) -> float:
70
+ return self.target + SLA_EPS - actual
71
72
@override
73
def format_cond(self, lhs: str) -> str:
@@ -71,8 +77,8 @@ class SLAGreaterThan(SLACriterionBase):
77
@dataclass
78
class SLAGreaterThanOrEqualTo(SLACriterionBase):
79
@override
74
- def validate(self, actual: float) -> bool:
75
- return actual >= self.target
80
+ def compute_margin(self, actual: float) -> float:
81
+ return self.target - actual
82
83
@override
84
def format_cond(self, lhs: str) -> str: