@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 77d9e514a2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[MISC] Replace input token throughput with total token throughput (#8164)

Co-authored-by: Michael Goin <michael@neuralmagic.com>

Cody Yu committed Sep 4, 2024 at 13:23 UTC 77d9e514a2284d5d0bd34b1518b9483ae7d8a05a
1 file changed +5 -5
benchmarks/benchmark_serving.py
+5 -5
@@ -56,8 +56,8 @@ class BenchmarkMetrics:
56 total_input: int
57 total_output: int
58 request_throughput: float
59 - input_throughput: float
59 output_throughput: float
60 + total_token_throughput: float
61 mean_ttft_ms: float
62 median_ttft_ms: float
63 std_ttft_ms: float
@@ -283,8 +283,8 @@ def calculate_metrics(
283 total_input=total_input,
284 total_output=sum(actual_output_lens),
285 request_throughput=completed / dur_s,
286 - input_throughput=total_input / dur_s,
286 output_throughput=sum(actual_output_lens) / dur_s,
287 + total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s,
288 mean_ttft_ms=np.mean(ttfts or 0) *
289 1000, # ttfts is empty if streaming is not supported by backend
290 std_ttft_ms=np.std(ttfts or 0) * 1000,
@@ -426,10 +426,10 @@ async def benchmark(
426 metrics.total_output))
427 print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
428 metrics.request_throughput))
429 - print("{:<40} {:<10.2f}".format("Input token throughput (tok/s):",
430 - metrics.input_throughput))
429 print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
430 metrics.output_throughput))
431 + print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):",
432 + metrics.total_token_throughput))
433
434 result = {
435 "duration": benchmark_duration,
@@ -437,8 +437,8 @@ async def benchmark(
437 "total_input_tokens": metrics.total_input,
438 "total_output_tokens": metrics.total_output,
439 "request_throughput": metrics.request_throughput,
440 - "input_throughput": metrics.input_throughput,
440 "output_throughput": metrics.output_throughput,
441 + "total_token_throughput": metrics.total_token_throughput,
442 "input_lens": [output.prompt_len for output in outputs],
443 "output_lens": actual_output_lens,
444 "ttfts": [output.ttft for output in outputs],