A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 10:45 UTC
[Benchmark] Add retry support to fix workload bias in multi-turn benchmark (#28493)
ai-jz committed
Nov 11, 2025 at 21:00 UTC
f31419ed8ba219d70a1982f238e42e74e0ed86c9
1 file changed
+82
-48
benchmarks/multi_turn/benchmark_serving_multi_turn.py
+82
-48
@@ -55,6 +55,7 @@ class ClientArgs(NamedTuple):
55
verify_output: bool
56
conversation_sampling: ConversationSampling
57
request_rate: float
58
+ max_retries: int
59
60
61
class RequestArgs(NamedTuple):
@@ -527,6 +528,25 @@ async def poisson_sleep(request_rate: float, verbose: bool = False) -> None:
528
await asyncio.sleep(interval)
529
530
531
+async def exponential_backoff_sleep(
532
+ attempt_cnt: int,
533
+ base_rate: float = 1.0,
534
+ backoff_factor: float = 2.0,
535
+ jitter_fraction: float = 0.10,
536
+ verbose: bool = False,
537
+) -> None:
538
+ # Sleep with exponential backoff and jitter after a failed request.
539
+ backoff_delay = base_rate * (backoff_factor**attempt_cnt)
540
+ jittered_delay = backoff_delay * (
541
+ 1 + np.random.uniform(-jitter_fraction, jitter_fraction)
542
+ )
543
+
544
+ if verbose:
545
+ logger.info(f"Backoff for {jittered_delay:.3f} seconds...")
546
+
547
+ await asyncio.sleep(jittered_delay)
548
+
549
+
550
async def client_main(
551
args: ClientArgs,
552
req_args: RequestArgs,
@@ -655,59 +675,62 @@ async def client_main(
675
)
676
time_of_last_turn[conv_id] = curr_time_sec
677
658
- success = True
659
- try:
660
- result = await send_turn(
661
- session,
662
- client_id,
663
- conv_id,
664
- messages,
665
- current_turn,
666
- tokenizer,
667
- req_args,
668
- args.print_content,
669
- args.verify_output,
670
- )
671
- if result is not None:
672
- result_queue.put(result)
673
- else:
674
- # None means that the request failed,
675
- # and should not be added to the statistics.
676
- success = False
677
- num_failures += 1
678
-
679
- logger.warning(
680
- f"{Color.YELLOW}Client {client_id} - Request rejected during conversation ID {conv_id} (turn: {current_turn}){Color.RESET}" # noqa: E501
678
+ success = False
679
+ for attempt_cnt in range(args.max_retries + 1):
680
+ try:
681
+ exception = False
682
+ result = await send_turn(
683
+ session,
684
+ client_id,
685
+ conv_id,
686
+ messages,
687
+ current_turn,
688
+ tokenizer,
689
+ req_args,
690
+ args.print_content,
691
+ args.verify_output,
692
+ )
693
+ if result is not None:
694
+ result_queue.put(result)
695
+ success = True
696
+ break
697
+ else:
698
+ logger.warning(
699
+ f"{Color.YELLOW}Client {client_id} - Request rejected during conversation ID {conv_id} (turn: {current_turn}){Color.RESET}" # noqa: E501
700
+ )
701
+ except asyncio.exceptions.TimeoutError:
702
+ exception = True
703
+ logger.error(
704
+ "%sClient %d - Timeout during conversation ID %s (turn: %d). "
705
+ "Base timeout is %ss (set with --request-timeout-sec), but the "
706
+ "effective timeout may be longer based on max_tokens. If this "
707
+ "is unexpected, consider increasing the timeout or checking "
708
+ "model performance.%s",
709
+ Color.RED,
710
+ client_id,
711
+ conv_id,
712
+ current_turn,
713
+ req_args.timeout_sec,
714
+ Color.RESET,
715
+ )
716
+ except Exception:
717
+ exception = True
718
+ logger.exception(
719
+ f"{Color.RED}Client {client_id} - Exception during conversation ID {conv_id} (turn: {current_turn}){Color.RESET}" # noqa: E501
720
)
721
683
- # Remove the conversation (should not be used again)
684
- active_convs.pop(conv_id)
685
-
686
- except asyncio.exceptions.TimeoutError:
687
- num_failures += 1
688
- logger.error(
689
- "%sClient %d - Timeout during conversation ID %s (turn: %d). "
690
- "Base timeout is %ss (set with --request-timeout-sec), but the "
691
- "effective timeout may be longer based on max_tokens. If this "
692
- "is unexpected, consider increasing the timeout or checking "
693
- "model performance.%s",
694
- Color.RED,
695
- client_id,
696
- conv_id,
697
- current_turn,
698
- req_args.timeout_sec,
699
- Color.RESET,
700
- )
701
- break # Exit gracefully instead of raising an error
722
+ # Sleep before retry if not last attempt
723
+ if not success and attempt_cnt < args.max_retries:
724
+ await exponential_backoff_sleep(attempt_cnt, verbose=args.verbose)
725
703
- except Exception:
726
+ if not success:
727
num_failures += 1
705
- logger.exception(
706
- f"{Color.RED}Client {client_id} - Exception during conversation ID {conv_id} (turn: {current_turn}){Color.RESET}" # noqa: E501
707
- )
708
- break # Exit gracefully instead of raising an error
728
+ # Remove the conversation (should not be used again)
729
+ active_convs.pop(conv_id)
730
+ if exception:
731
+ break # Exit gracefully instead of raising an error
732
710
- if success:
733
+ else:
734
num_successes += 1
735
736
# Update the turns counter to include the LLM response
@@ -822,6 +845,7 @@ def get_client_config(
845
verify_output=args.verify_output,
846
conversation_sampling=args.conversation_sampling,
847
request_rate=args.request_rate,
848
+ max_retries=args.max_retries,
849
)
850
851
if args.limit_min_tokens > 0 or args.limit_max_tokens > 0:
@@ -1357,6 +1381,16 @@ async def main() -> None:
1381
help="Expected request rate (Poisson process) per client in requests/sec."
1382
"Set to 0 for no delay between requests.",
1383
)
1384
+ parser.add_argument(
1385
+ "--max-retries",
1386
+ type=int,
1387
+ default=int(os.environ.get("MULTITURN_BENCH_MAX_RETRIES", "0")),
1388
+ help="Maximum number of retry attempts for timed-out requests. "
1389
+ "Default is 0 (no retries). "
1390
+ "Set to higher values to retry failed requests and maintain "
1391
+ "fair workload distribution. "
1392
+ "Can also be set via MULTITURN_BENCH_MAX_RETRIES environment variable.",
1393
+ )
1394
parser.add_argument(
1395
"--conversation-sampling",
1396
type=ConversationSampling,