A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 12:00 UTC
Add request timeout override for multi-turn benchmarks (#28386)
Signed-off-by: Ido Segev <idos@pliops.com>
Ido Segev committed
Nov 11, 2025 at 15:41 UTC
3380543b2075abd6f3e6e283f4eacb307354e33a
1 file changed
+35
-5
benchmarks/multi_turn/benchmark_serving_multi_turn.py
+35
-5
@@ -63,6 +63,7 @@ class RequestArgs(NamedTuple):
63
stream: bool
64
limit_min_tokens: int # Use negative value for no limit
65
limit_max_tokens: int # Use negative value for no limit
66
+ timeout_sec: int
67
68
69
class BenchmarkArgs(NamedTuple):
@@ -214,6 +215,7 @@ async def send_request(
215
stream: bool = True,
216
min_tokens: int | None = None,
217
max_tokens: int | None = None,
218
+ timeout_sec: int = 120,
219
) -> ServerResponse:
220
payload = {
221
"model": model,
@@ -235,10 +237,16 @@ async def send_request(
237
headers = {"Content-Type": "application/json"}
238
239
# Calculate the timeout for the request
238
- timeout_sec = 120
240
if max_tokens is not None:
241
# Assume TPOT of 200ms and use max_tokens to determine timeout
241
- timeout_sec = max(timeout_sec, int(max_tokens * 0.2))
242
+ token_based_timeout = int(max_tokens * 0.2)
243
+ if token_based_timeout > timeout_sec:
244
+ timeout_sec = token_based_timeout
245
+ logger.info(
246
+ "Using timeout of %ds based on max_tokens %d",
247
+ timeout_sec,
248
+ max_tokens,
249
+ )
250
timeout = aiohttp.ClientTimeout(total=timeout_sec)
251
252
valid_response = True
@@ -409,6 +417,7 @@ async def send_turn(
417
req_args.stream,
418
min_tokens,
419
max_tokens,
420
+ req_args.timeout_sec,
421
)
422
423
if response.valid is False:
@@ -676,8 +685,18 @@ async def client_main(
685
686
except asyncio.exceptions.TimeoutError:
687
num_failures += 1
679
- logger.exception(
680
- f"{Color.RED}Client {client_id} - Timeout during conversation ID {conv_id} (turn: {current_turn}){Color.RESET}" # noqa: E501
688
+ logger.error(
689
+ "%sClient %d - Timeout during conversation ID %s (turn: %d). "
690
+ "Base timeout is %ss (set with --request-timeout-sec), but the "
691
+ "effective timeout may be longer based on max_tokens. If this "
692
+ "is unexpected, consider increasing the timeout or checking "
693
+ "model performance.%s",
694
+ Color.RED,
695
+ client_id,
696
+ conv_id,
697
+ current_turn,
698
+ req_args.timeout_sec,
699
+ Color.RESET,
700
)
701
break # Exit gracefully instead of raising an error
702
@@ -815,6 +834,9 @@ def get_client_config(
834
"Invalid min/max tokens limits (min should not be larger than max)"
835
)
836
837
+ if args.request_timeout_sec <= 0:
838
+ raise ValueError("Request timeout must be a positive number")
839
+
840
# Arguments for API requests
841
chat_url = f"{args.url}/v1/chat/completions"
842
model_name = args.served_model_name if args.served_model_name else args.model
@@ -825,6 +847,7 @@ def get_client_config(
847
stream=not args.no_stream,
848
limit_min_tokens=args.limit_min_tokens,
849
limit_max_tokens=args.limit_max_tokens,
850
+ timeout_sec=args.request_timeout_sec,
851
)
852
853
return client_args, req_args
@@ -968,7 +991,7 @@ async def main_mp(
991
f"(is alive: {client.is_alive()}){Color.RESET}"
992
)
993
971
- client.join(timeout=120)
994
+ client.join(timeout=req_args.timeout_sec + 1)
995
996
if client.is_alive():
997
logger.warning(
@@ -1351,6 +1374,13 @@ async def main() -> None:
1374
action="store_true",
1375
help="Verify the LLM output (compare to the answers in the input JSON file)",
1376
)
1377
+ parser.add_argument(
1378
+ "--request-timeout-sec",
1379
+ type=int,
1380
+ default=120,
1381
+ help="Timeout in seconds for each API request (default: 120). "
1382
+ "Automatically increased if max tokens imply longer decoding.",
1383
+ )
1384
1385
parser.add_argument(
1386
"--no-stream",