A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Misc] Separate total and output tokens in benchmark_throughput.py (#8914)
Michael Goin committed
Oct 23, 2024 at 12:47 UTC
fd0e2cfdb2e0fa6ee2822a73141441de51114f2a
1 file changed
+3
-1
benchmarks/benchmark_throughput.py
+3
-1
@@ -272,8 +272,10 @@ def main(args: argparse.Namespace):
272
raise ValueError(f"Unknown backend: {args.backend}")
273
total_num_tokens = sum(prompt_len + output_len
274
for _, prompt_len, output_len in requests)
275
+ total_output_tokens = sum(output_len for _, _, output_len in requests)
276
print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
276
- f"{total_num_tokens / elapsed_time:.2f} tokens/s")
277
+ f"{total_num_tokens / elapsed_time:.2f} total tokens/s, "
278
+ f"{total_output_tokens / elapsed_time:.2f} output tokens/s")
279
280
# Output JSON results if specified
281
if args.output_json: