A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[MISC] Replace input token throughput with total token throughput (#8164)
Co-authored-by: Michael Goin <michael@neuralmagic.com>
Cody Yu committed
Sep 4, 2024 at 13:23 UTC
77d9e514a2284d5d0bd34b1518b9483ae7d8a05a
1 file changed
+5
-5
benchmarks/benchmark_serving.py
+5
-5
@@ -56,8 +56,8 @@ class BenchmarkMetrics:
56
total_input: int
57
total_output: int
58
request_throughput: float
59
- input_throughput: float
59
output_throughput: float
60
+ total_token_throughput: float
61
mean_ttft_ms: float
62
median_ttft_ms: float
63
std_ttft_ms: float
@@ -283,8 +283,8 @@ def calculate_metrics(
283
total_input=total_input,
284
total_output=sum(actual_output_lens),
285
request_throughput=completed / dur_s,
286
- input_throughput=total_input / dur_s,
286
output_throughput=sum(actual_output_lens) / dur_s,
287
+ total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s,
288
mean_ttft_ms=np.mean(ttfts or 0) *
289
1000, # ttfts is empty if streaming is not supported by backend
290
std_ttft_ms=np.std(ttfts or 0) * 1000,
@@ -426,10 +426,10 @@ async def benchmark(
426
metrics.total_output))
427
print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
428
metrics.request_throughput))
429
- print("{:<40} {:<10.2f}".format("Input token throughput (tok/s):",
430
- metrics.input_throughput))
429
print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
430
metrics.output_throughput))
431
+ print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):",
432
+ metrics.total_token_throughput))
433
434
result = {
435
"duration": benchmark_duration,
@@ -437,8 +437,8 @@ async def benchmark(
437
"total_input_tokens": metrics.total_input,
438
"total_output_tokens": metrics.total_output,
439
"request_throughput": metrics.request_throughput,
440
- "input_throughput": metrics.input_throughput,
440
"output_throughput": metrics.output_throughput,
441
+ "total_token_throughput": metrics.total_token_throughput,
442
"input_lens": [output.prompt_len for output in outputs],
443
"output_lens": actual_output_lens,
444
"ttfts": [output.ttft for output in outputs],