[Bugfix] Account for client queue time in serve benchmarks (#54136)
Signed-off-by: maithilijoshi20 <maithilijoshi2003@gmail.com> Signed-off-by: maithilijoshi20 <97733343+maithilijoshi20@users.noreply.github.com> Signed-off-by: maithilijoshi20 <maithilij2003@gmail.com> Co-authored-by: maithilijoshi20 <maithilijoshi2003@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
maithilijoshi20 committed
Sep 1, 2026 at 19:40 UTC
9e905f7450fb556c4f43e49d4ece0f728cbfda2b
3 files changed
+64
-27
docs/benchmarking/cli.md
+1
-1
@@ -664,7 +664,7 @@ vLLM's benchmark serving script provides sophisticated load pattern simulation c
664
665
- `--request-rate`: Controls the target request generation rate (requests per second). Set to `inf` for maximum throughput testing or finite values for controlled load simulation.
666
- `--burstiness`: Controls traffic variability using a Gamma distribution (range: > 0). Lower values create bursty traffic, higher values create uniform traffic.
667
-- `--max-concurrency`: Limits concurrent outstanding requests. If this argument is not provided, concurrency is unlimited. Set a value to simulate backpressure.
667
+- `--max-concurrency`: Limits concurrent outstanding requests. If this argument is not provided, concurrency is unlimited. Set a value to simulate backpressure. When set, include `client_queue_time` in `--percentile-metrics` to report time spent waiting for the benchmark client's concurrency limit. With a finite `--request-rate`, `e2el_including_client_queue` reports schedule-relative end-to-end latency; it is omitted for `--request-rate=inf`, where all requests arrive at benchmark start.
668
669
These parameters work together to create realistic load patterns with carefully chosen defaults. The `--request-rate` parameter defaults to `inf` (infinite), which sends all requests immediately for maximum throughput testing. When set to finite values, it uses either a Poisson process (default `--burstiness=1.0`) or Gamma distribution for realistic request timing. The `--burstiness` parameter only takes effect when `--request-rate` is not infinite - a value of 1.0 creates natural Poisson traffic, while lower values (0.1-0.5) create bursty patterns and higher values (2.0-5.0) create uniform spacing. The `--max-concurrency` parameter defaults to `None` (unlimited) but can be set to simulate real-world constraints where a load balancer or API gateway limits concurrent connections. When combined, these parameters allow you to simulate everything from unrestricted stress testing (`--request-rate=inf`) to production-like scenarios with realistic arrival patterns and resource constraints.
670
vllm/benchmarks/lib/endpoint_request_func.py
+2
@@ -99,6 +99,8 @@ class RequestFuncOutput:
99
prompt_len: int = 0
100
error: str = ""
101
start_time: float = 0.0
102
+ # Time spent awaiting the benchmark client concurrency semaphore.
103
+ client_queue_time: float = 0.0
104
input_audio_duration: float = 0.0 # in seconds
105
num_input_sequences: int = 1
106
vllm/benchmarks/serve.py
+61
-26
@@ -23,6 +23,7 @@ import asyncio
23
import contextlib
24
import importlib.util
25
import json
26
+import math
27
import os
28
import random
29
import shutil
@@ -972,11 +973,17 @@ async def benchmark(
973
else contextlib.nullcontext()
974
)
975
975
- async def limited_request_func(request_func_input, session, pbar):
976
+ async def limited_request_func(
977
+ request_func_input, session, pbar, request_arrival_time
978
+ ):
979
async with semaphore:
977
- return await request_func(
980
+ output = await request_func(
981
request_func_input=request_func_input, session=session, pbar=pbar
982
)
983
+ # Preserve start_time as the time the request was sent. It is used by
984
+ # throughput calculations; record client-side semaphore delay separately.
985
+ output.client_queue_time = output.start_time - request_arrival_time
986
+ return output
987
988
probe_outputs: list[RequestFuncOutput] = []
989
probe_stop = asyncio.Event()
@@ -1064,10 +1071,11 @@ async def benchmark(
1071
request_id=request_id,
1072
chat_messages=request.chat_messages,
1073
)
1074
+ request_arrival_time = time.perf_counter()
1075
tasks.append(
1076
asyncio.create_task(
1077
limited_request_func(
1070
- request_func_input=request_func_input, session=session, pbar=pbar
1078
+ request_func_input, session, pbar, request_arrival_time
1079
)
1080
)
1081
)
@@ -1284,7 +1292,9 @@ async def benchmark(
1292
"output_lens": actual_output_lens,
1293
"ttfts": [output.ttft for output in outputs],
1294
"itls": [output.itl for output in outputs],
1295
+ "latencies": [output.latency for output in outputs],
1296
"start_times": [output.start_time for output in outputs],
1297
+ "queue_times": [output.client_queue_time for output in outputs],
1298
"generated_texts": [output.generated_text for output in outputs],
1299
"errors": [output.error for output in outputs],
1300
"max_output_tokens_per_s": metrics.max_output_tokens_per_s,
@@ -1301,9 +1311,22 @@ async def benchmark(
1311
"input_sequence_throughput": metrics.input_sequence_throughput,
1312
"total_token_throughput": metrics.total_token_throughput,
1313
"input_lens": [output.prompt_len for output in outputs],
1314
+ "latencies": [output.latency for output in outputs],
1315
+ "queue_times": [output.client_queue_time for output in outputs],
1316
"errors": [output.error for output in outputs],
1317
}
1318
1319
+ queue_times: list[float] | None = None
1320
+ e2els_including_queue: list[float] | None = None
1321
+ if max_concurrency is not None:
1322
+ queue_times = [output.client_queue_time for output in outputs if output.success]
1323
+ if not math.isinf(request_rate):
1324
+ e2els_including_queue = [
1325
+ output.latency + output.client_queue_time
1326
+ for output in outputs
1327
+ if output.success
1328
+ ]
1329
+
1330
if probe_stats is not None:
1331
result.update(probe_stats)
1332
@@ -1339,34 +1362,31 @@ async def benchmark(
1362
metric_name: str,
1363
# E.g., "Time to First Token"
1364
metric_header: str,
1365
+ values: list[float] | None = None,
1366
):
1367
# This function prints and adds statistics of the specified
1368
# metric.
1369
if metric_attribute_name not in selected_percentile_metrics:
1370
return
1371
+ if values is None:
1372
+ mean = getattr(metrics, f"mean_{metric_attribute_name}_ms")
1373
+ median = getattr(metrics, f"median_{metric_attribute_name}_ms")
1374
+ std = getattr(metrics, f"std_{metric_attribute_name}_ms")
1375
+ percentiles = getattr(metrics, f"percentiles_{metric_attribute_name}_ms")
1376
+ else:
1377
+ mean = np.mean(values or 0) * 1000
1378
+ median = np.median(values or 0) * 1000
1379
+ std = np.std(values or 0) * 1000
1380
+ percentiles = [
1381
+ (p, np.percentile(values or 0, p) * 1000) for p in selected_percentiles
1382
+ ]
1383
print("{s:{c}^{n}}".format(s=metric_header, n=50, c="-"))
1348
- print(
1349
- "{:<40} {:<10.2f}".format(
1350
- f"Mean {metric_name} (ms):",
1351
- getattr(metrics, f"mean_{metric_attribute_name}_ms"),
1352
- )
1353
- )
1354
- print(
1355
- "{:<40} {:<10.2f}".format(
1356
- f"Median {metric_name} (ms):",
1357
- getattr(metrics, f"median_{metric_attribute_name}_ms"),
1358
- )
1359
- )
1360
- result[f"mean_{metric_attribute_name}_ms"] = getattr(
1361
- metrics, f"mean_{metric_attribute_name}_ms"
1362
- )
1363
- result[f"median_{metric_attribute_name}_ms"] = getattr(
1364
- metrics, f"median_{metric_attribute_name}_ms"
1365
- )
1366
- result[f"std_{metric_attribute_name}_ms"] = getattr(
1367
- metrics, f"std_{metric_attribute_name}_ms"
1368
- )
1369
- for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}_ms"):
1384
+ print("{:<40} {:<10.2f}".format(f"Mean {metric_name} (ms):", mean))
1385
+ print("{:<40} {:<10.2f}".format(f"Median {metric_name} (ms):", median))
1386
+ result[f"mean_{metric_attribute_name}_ms"] = mean
1387
+ result[f"median_{metric_attribute_name}_ms"] = median
1388
+ result[f"std_{metric_attribute_name}_ms"] = std
1389
+ for p, value in percentiles:
1390
p_word = str(int(p)) if int(p) == p else str(p)
1391
print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):", value))
1392
result[f"p{p_word}_{metric_attribute_name}_ms"] = value
@@ -1376,6 +1396,20 @@ async def benchmark(
1396
process_one_metric("tpot", "TPOT", "Time per Output Token (excl. 1st token)")
1397
process_one_metric("itl", "ITL", "Inter-token Latency")
1398
process_one_metric("e2el", "E2EL", "End-to-end Latency")
1399
+ if queue_times is not None:
1400
+ process_one_metric(
1401
+ "client_queue_time",
1402
+ "Client Queue Time",
1403
+ "Client-side Queueing",
1404
+ queue_times,
1405
+ )
1406
+ if e2els_including_queue is not None:
1407
+ process_one_metric(
1408
+ "e2el_including_client_queue",
1409
+ "E2EL incl. Client Queue",
1410
+ "Queue-inclusive End-to-end Latency",
1411
+ e2els_including_queue,
1412
+ )
1413
1414
if diffusion_stats is not None:
1415
print("{s:{c}^{n}}".format(s="Diffusion Decoding", n=50, c="-"))
@@ -1776,7 +1810,8 @@ def add_cli_args(parser: FlexibleArgumentParser):
1810
default=None,
1811
help="Comma-separated list of selected metrics to report percentiles. "
1812
"This argument specifies the metrics to report percentiles. "
1779
- 'Allowed metric names are "ttft", "tpot", "itl", "e2el". '
1813
+ 'Allowed metric names are "ttft", "tpot", "itl", "e2el", '
1814
+ '"client_queue_time", "e2el_including_client_queue". '
1815
'If not specified, defaults to "ttft,tpot,itl" for generative models '
1816
'and "e2el" for pooling models.',
1817
)