A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 20:45 UTC
[Feature][Benchmarks] Support `inf` burstiness (#26941)
Signed-off-by: Sophie du Couédic <sop@zurich.ibm.com>
Sophie du Couédic committed
Nov 3, 2025 at 19:33 UTC
a4398fbb5e9fe20c8f0f092da4de30c9a582cce0
1 file changed
+7
vllm/benchmarks/serve.py
+7
@@ -189,9 +189,16 @@ async def get_request(
189
total_requests,
190
request_rate,
191
)
192
+ assert current_request_rate > 0.0, (
193
+ f"Obtained non-positive request rate {current_request_rate}."
194
+ )
195
request_rates.append(current_request_rate)
196
if current_request_rate == float("inf"):
197
delay_ts.append(0)
198
+ elif burstiness == float("inf"):
199
+ # when burstiness tends to infinity, the delay time becomes constant
200
+ # and tends to the inverse of the request rate
201
+ delay_ts.append(1.0 / current_request_rate)
202
else:
203
theta = 1.0 / (current_request_rate * burstiness)
204