A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 06:00 UTC
[Misc] Multi-turn benchmark output performance json (#39572)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Apr 13, 2026 at 20:15 UTC
8625ec267b35a158721467e81d5a74ee53091a73
1 file changed
+19
benchmarks/multi_turn/benchmark_serving_multi_turn.py
+19
@@ -1439,6 +1439,12 @@ async def main() -> None:
1439
action="store_true",
1440
help="Export summary to Excel file (optional)",
1441
)
1442
+ parser.add_argument(
1443
+ "--stats-json-output",
1444
+ type=str,
1445
+ default=None,
1446
+ help="Export per-request stats (ttft_ms, tpot_ms, etc.) to a JSON file",
1447
+ )
1448
parser.add_argument(
1449
"-v",
1450
"--verbose",
@@ -1651,6 +1657,19 @@ async def main() -> None:
1657
warmup_runtime_sec=warmup_runtime_sec,
1658
)
1659
1660
+ if args.stats_json_output is not None:
1661
+ # Export per-request metrics as a JSON array for downstream analysis.
1662
+ stats_data = [s._asdict() for s in client_metrics]
1663
+ logger.info(
1664
+ f"{Color.GREEN}Writing per-request stats JSON: "
1665
+ f"{args.stats_json_output}{Color.RESET}"
1666
+ )
1667
+ os.makedirs(
1668
+ os.path.dirname(os.path.abspath(args.stats_json_output)), exist_ok=True
1669
+ )
1670
+ with open(args.stats_json_output, "w") as f:
1671
+ json.dump(stats_data, f, indent=2)
1672
+
1673
if args.output_file is not None:
1674
# Write a JSON file with the updated conversations
1675
# The "assistant" content will contain the answers from the tested LLM