[Benchmark] Add probe requests to vllm bench serve (#49611)
Signed-off-by: Guan-Ming (Wesley) Chiu <105915352+guan404ming@users.noreply.github.com>
Guan-Ming Chiu committed
Aug 1, 2026 at 23:10 UTC
63e78ce3652f4f94e9f484f40db71ca4cf019f21
3 files changed
+106
docs/benchmarking/cli.md
+25
@@ -622,6 +622,31 @@ The following arguments can be used to control the ramp-up:
622
- `--ramp-up-start-rps`: The request rate at the beginning of the benchmark.
623
- `--ramp-up-end-rps`: The request rate at the end of the benchmark.
624
625
+#### Probe Requests
626
+
627
+The benchmark tool also supports sending probe requests alongside the main
628
+workload. This can be useful for measuring how the main workload affects
629
+unrelated traffic sharing the server, e.g. a few requests with large images
630
+stalling a concurrent lightweight request while their multimodal preprocessing
631
+occupies the frontend.
632
+
633
+Setting `--probe-request-rate` to a positive value sends single-token text-only
634
+probe requests at that rate (requests per second) alongside the main workload.
635
+Probes bypass `--max-concurrency` and their latency is reported separately, so
636
+the probe percentiles directly measure the interference that the main workload
637
+inflicts on unrelated requests.
638
+
639
+```bash
640
+vllm bench serve \
641
+ --model Qwen/Qwen2.5-VL-3B-Instruct \
642
+ --backend openai-chat \
643
+ --endpoint /v1/chat/completions \
644
+ --dataset-name random-mm \
645
+ --random-mm-bucket-config '{(2048, 2048, 1): 1.0}' \
646
+ --request-rate 4 \
647
+ --probe-request-rate 20
648
+```
649
+
650
#### Load Pattern Configuration
651
652
vLLM's benchmark serving script provides sophisticated load pattern simulation capabilities through three key parameters that control request generation and concurrency behavior:
tests/benchmarks/test_skip_tokenizer_init.py
+1
@@ -84,6 +84,7 @@ def _args(dataset_path: str) -> argparse.Namespace:
84
request_rate=16.0,
85
burstiness=1.0,
86
max_concurrency=None,
87
+ probe_request_rate=0.0,
88
# misc serve args that main_async reads before reaching get_samples
89
plot_timeline=False,
90
plot_dataset_stats=False,
vllm/benchmarks/serve.py
+80
@@ -795,6 +795,7 @@ async def benchmark(
795
ready_check_timeout_sec: int = 600,
796
ssl_context: ssl.SSLContext | bool | None = None,
797
self_timed: bool = False,
798
+ probe_request_rate: float = 0.0,
799
):
800
try:
801
request_func = ASYNC_REQUEST_FUNCS[endpoint_type]
@@ -971,6 +972,30 @@ async def benchmark(
972
request_func_input=request_func_input, session=session, pbar=pbar
973
)
974
975
+ probe_outputs: list[RequestFuncOutput] = []
976
+ probe_stop = asyncio.Event()
977
+
978
+ async def probe_loop():
979
+ probe_input = replace(
980
+ test_input,
981
+ prompt="Hi",
982
+ prompt_len=1,
983
+ output_len=1,
984
+ multi_modal_content=None,
985
+ chat_messages=None,
986
+ )
987
+ interval = 1 / probe_request_rate
988
+ while not probe_stop.is_set():
989
+ probe_outputs.append(
990
+ await request_func(request_func_input=probe_input, session=session)
991
+ )
992
+ await asyncio.sleep(interval)
993
+
994
+ probe_task: asyncio.Task | None = None
995
+ if probe_request_rate > 0:
996
+ print(f"Probe request rate: {probe_request_rate} req/s")
997
+ probe_task = asyncio.create_task(probe_loop())
998
+
999
benchmark_start_time = time.perf_counter()
1000
tasks: list[asyncio.Task] = []
1001
@@ -1042,6 +1067,10 @@ async def benchmark(
1067
)
1068
outputs: list[RequestFuncOutput] = await asyncio.gather(*tasks)
1069
1070
+ if probe_task is not None:
1071
+ probe_stop.set()
1072
+ await probe_task
1073
+
1074
if pbar is not None:
1075
pbar.close()
1076
@@ -1189,6 +1218,44 @@ async def benchmark(
1218
)
1219
)
1220
1221
+ probe_stats: dict[str, Any] | None = None
1222
+ if probe_task is not None:
1223
+ probe_lats = [o.latency for o in probe_outputs if o.success]
1224
+ if probe_lats:
1225
+ probe_stats = {
1226
+ "probe_completed": len(probe_lats),
1227
+ "probe_failed": len(probe_outputs) - len(probe_lats),
1228
+ "probe_median_e2el_ms": float(np.median(probe_lats)) * 1000,
1229
+ "probe_p99_e2el_ms": float(np.percentile(probe_lats, 99)) * 1000,
1230
+ "probe_max_e2el_ms": float(max(probe_lats)) * 1000,
1231
+ }
1232
+ print("{s:{c}^{n}}".format(s="Probe Requests", n=50, c="-"))
1233
+ print(
1234
+ "{:<40} {:<10}".format(
1235
+ "Probe requests completed:", probe_stats["probe_completed"]
1236
+ )
1237
+ )
1238
+ print(
1239
+ "{:<40} {:<10}".format(
1240
+ "Probe requests failed:", probe_stats["probe_failed"]
1241
+ )
1242
+ )
1243
+ print(
1244
+ "{:<40} {:<10.2f}".format(
1245
+ "Median probe E2EL (ms):", probe_stats["probe_median_e2el_ms"]
1246
+ )
1247
+ )
1248
+ print(
1249
+ "{:<40} {:<10.2f}".format(
1250
+ "P99 probe E2EL (ms):", probe_stats["probe_p99_e2el_ms"]
1251
+ )
1252
+ )
1253
+ print(
1254
+ "{:<40} {:<10.2f}".format(
1255
+ "Max probe E2EL (ms):", probe_stats["probe_max_e2el_ms"]
1256
+ )
1257
+ )
1258
+
1259
result: dict[str, Any]
1260
if isinstance(metrics, BenchmarkMetrics):
1261
result = {
@@ -1223,6 +1290,9 @@ async def benchmark(
1290
"errors": [output.error for output in outputs],
1291
}
1292
1293
+ if probe_stats is not None:
1294
+ result.update(probe_stats)
1295
+
1296
if rps_change_events:
1297
result["rps_change_events"] = rps_change_events
1298
@@ -1605,6 +1675,15 @@ def add_cli_args(parser: FlexibleArgumentParser):
1675
"bursty requests. A higher burstiness value (burstiness > 1) "
1676
"results in a more uniform arrival of requests.",
1677
)
1678
+ parser.add_argument(
1679
+ "--probe-request-rate",
1680
+ type=float,
1681
+ default=0.0,
1682
+ help="If positive, send single-token text-only probe requests at "
1683
+ "this rate (req/s) alongside the main workload, bypassing "
1684
+ "--max-concurrency, and report their latency separately. Useful "
1685
+ "for measuring how the main workload stalls unrelated requests.",
1686
+ )
1687
parser.add_argument(
1688
"--disable-tqdm",
1689
action="store_true",
@@ -2121,6 +2200,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
2200
ready_check_timeout_sec=args.ready_check_timeout_sec,
2201
ssl_context=ssl_context,
2202
self_timed=args.self_timed,
2203
+ probe_request_rate=args.probe_request_rate,
2204
)
2205
2206
# Save config and results to json