@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fd0e2cfdb2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Misc] Separate total and output tokens in benchmark_throughput.py (#8914)

Michael Goin committed Oct 23, 2024 at 12:47 UTC fd0e2cfdb2e0fa6ee2822a73141441de51114f2a
1 file changed +3 -1
benchmarks/benchmark_throughput.py
+3 -1
@@ -272,8 +272,10 @@ def main(args: argparse.Namespace):
272 raise ValueError(f"Unknown backend: {args.backend}")
273 total_num_tokens = sum(prompt_len + output_len
274 for _, prompt_len, output_len in requests)
275 + total_output_tokens = sum(output_len for _, _, output_len in requests)
276 print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
276 - f"{total_num_tokens / elapsed_time:.2f} tokens/s")
277 + f"{total_num_tokens / elapsed_time:.2f} total tokens/s, "
278 + f"{total_output_tokens / elapsed_time:.2f} output tokens/s")
279
280 # Output JSON results if specified
281 if args.output_json: