A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 22:35 UTC
[benchmark] add max-concurrency in result table (#21095)
Signed-off-by: Peter Pan <Peter.Pan@daocloud.io>
Peter Pan committed
Jul 30, 2025 at 16:15 UTC
533db0935da051ac793e8b22afbcb9ae9fa4255b
3 files changed
+14
benchmarks/benchmark_serving.py
+4
@@ -413,6 +413,10 @@ async def benchmark(
413
414
print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
415
print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
416
+ if max_concurrency is not None:
417
+ print("{:<40} {:<10}".format("Maximum request concurrency:", max_concurrency))
418
+ if request_rate != float("inf"):
419
+ print("{:<40} {:<10.2f}".format("Request rate configured (RPS):", request_rate))
420
print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration))
421
print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
422
print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output))
benchmarks/benchmark_serving_structured_output.py
+4
@@ -555,6 +555,10 @@ async def benchmark(
555
556
print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="="))
557
print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
558
+ if max_concurrency is not None:
559
+ print("{:<40} {:<10}".format("Maximum request concurrency:", max_concurrency))
560
+ if request_rate != float("inf"):
561
+ print("{:<40} {:<10.2f}".format("Request rate configured (RPS):", request_rate))
562
print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration))
563
print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
564
print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output))
vllm/benchmarks/serve.py
+6
@@ -486,6 +486,12 @@ async def benchmark(
486
487
print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
488
print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
489
+ if max_concurrency is not None:
490
+ print("{:<40} {:<10}".format("Maximum request concurrency:",
491
+ max_concurrency))
492
+ if request_rate != float('inf'):
493
+ print("{:<40} {:<10.2f}".format("Request rate configured (RPS):",
494
+ request_rate ))
495
print("{:<40} {:<10.2f}".format("Benchmark duration (s):",
496
benchmark_duration))
497
print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))