A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
Add more percentiles and latencies (#7759)
Wei-Sheng Chin committed
Aug 29, 2024 at 16:48 UTC
0c785d344db23644139940d19d5c448754ef53d7
1 file changed
+94
-38
benchmarks/benchmark_serving.py
+94
-38
@@ -61,15 +61,22 @@ class BenchmarkMetrics:
61
mean_ttft_ms: float
62
median_ttft_ms: float
63
std_ttft_ms: float
64
- p99_ttft_ms: float
64
+ percentiles_ttft_ms: List[Tuple[float, float]]
65
mean_tpot_ms: float
66
median_tpot_ms: float
67
std_tpot_ms: float
68
- p99_tpot_ms: float
68
+ percentiles_tpot_ms: List[Tuple[float, float]]
69
mean_itl_ms: float
70
median_itl_ms: float
71
std_itl_ms: float
72
- p99_itl_ms: float
72
+ percentiles_itl_ms: List[Tuple[float, float]]
73
+ # E2EL stands for end-to-end latency per request.
74
+ # It is the time taken on the client side from sending
75
+ # a request to receiving a complete response.
76
+ mean_e2el_ms: float
77
+ median_e2el_ms: float
78
+ std_e2el_ms: float
79
+ percentiles_e2el_ms: List[Tuple[float, float]]
80
81
82
def sample_sharegpt_requests(
@@ -235,6 +242,8 @@ def calculate_metrics(
242
outputs: List[RequestFuncOutput],
243
dur_s: float,
244
tokenizer: PreTrainedTokenizerBase,
245
+ selected_percentile_metrics: List[str],
246
+ selected_percentiles: List[float],
247
) -> Tuple[BenchmarkMetrics, List[int]]:
248
actual_output_lens: List[int] = []
249
total_input = 0
@@ -242,6 +251,7 @@ def calculate_metrics(
251
itls: List[float] = []
252
tpots: List[float] = []
253
ttfts: List[float] = []
254
+ e2els: List[float] = []
255
for i in range(len(outputs)):
256
if outputs[i].success:
257
# We use the tokenizer to count the number of output tokens for all
@@ -258,6 +268,7 @@ def calculate_metrics(
268
(outputs[i].latency - outputs[i].ttft) / (output_len - 1))
269
itls += outputs[i].itl
270
ttfts.append(outputs[i].ttft)
271
+ e2els.append(outputs[i].latency)
272
completed += 1
273
else:
274
actual_output_lens.append(0)
@@ -276,17 +287,25 @@ def calculate_metrics(
287
output_throughput=sum(actual_output_lens) / dur_s,
288
mean_ttft_ms=np.mean(ttfts or 0) *
289
1000, # ttfts is empty if streaming is not supported by backend
279
- median_ttft_ms=np.median(ttfts or 0) * 1000,
290
std_ttft_ms=np.std(ttfts or 0) * 1000,
281
- p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000,
291
+ median_ttft_ms=np.median(ttfts or 0) * 1000,
292
+ percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000)
293
+ for p in selected_percentiles],
294
mean_tpot_ms=np.mean(tpots or 0) * 1000,
283
- median_tpot_ms=np.median(tpots or 0) * 1000,
295
std_tpot_ms=np.std(tpots or 0) * 1000,
285
- p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000,
296
+ median_tpot_ms=np.median(tpots or 0) * 1000,
297
+ percentiles_tpot_ms=[(p, np.percentile(tpots or 0, p) * 1000)
298
+ for p in selected_percentiles],
299
mean_itl_ms=np.mean(itls or 0) * 1000,
287
- median_itl_ms=np.median(itls or 0) * 1000,
300
std_itl_ms=np.std(itls or 0) * 1000,
289
- p99_itl_ms=np.percentile(itls or 0, 99) * 1000,
301
+ median_itl_ms=np.median(itls or 0) * 1000,
302
+ percentiles_itl_ms=[(p, np.percentile(itls or 0, p) * 1000)
303
+ for p in selected_percentiles],
304
+ mean_e2el_ms=np.median(e2els or 0) * 1000,
305
+ std_e2el_ms=np.std(e2els or 0) * 1000,
306
+ median_e2el_ms=np.mean(e2els or 0) * 1000,
307
+ percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000)
308
+ for p in selected_percentiles],
309
)
310
311
return metrics, actual_output_lens
@@ -304,6 +323,8 @@ async def benchmark(
323
request_rate: float,
324
disable_tqdm: bool,
325
profile: bool,
326
+ selected_percentile_metrics: List[str],
327
+ selected_percentiles: List[str],
328
):
329
if backend in ASYNC_REQUEST_FUNCS:
330
request_func = ASYNC_REQUEST_FUNCS[backend]
@@ -392,6 +413,8 @@ async def benchmark(
413
outputs=outputs,
414
dur_s=benchmark_duration,
415
tokenizer=tokenizer,
416
+ selected_percentile_metrics=selected_percentile_metrics,
417
+ selected_percentiles=selected_percentiles,
418
)
419
420
print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
@@ -407,23 +430,6 @@ async def benchmark(
430
metrics.input_throughput))
431
print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
432
metrics.output_throughput))
410
- print("{s:{c}^{n}}".format(s='Time to First Token', n=50, c='-'))
411
- print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms))
412
- print("{:<40} {:<10.2f}".format("Median TTFT (ms):",
413
- metrics.median_ttft_ms))
414
- print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms))
415
- print("{s:{c}^{n}}".format(s='Time per Output Token (excl. 1st token)',
416
- n=50,
417
- c='-'))
418
- print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms))
419
- print("{:<40} {:<10.2f}".format("Median TPOT (ms):",
420
- metrics.median_tpot_ms))
421
- print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms))
422
- print("{s:{c}^{n}}".format(s='Inter-token Latency', n=50, c='-'))
423
- print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms))
424
- print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms))
425
- print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms))
426
- print("=" * 50)
433
434
result = {
435
"duration": benchmark_duration,
@@ -433,18 +439,6 @@ async def benchmark(
439
"request_throughput": metrics.request_throughput,
440
"input_throughput": metrics.input_throughput,
441
"output_throughput": metrics.output_throughput,
436
- "mean_ttft_ms": metrics.mean_ttft_ms,
437
- "median_ttft_ms": metrics.median_ttft_ms,
438
- "std_ttft_ms": metrics.std_ttft_ms,
439
- "p99_ttft_ms": metrics.p99_ttft_ms,
440
- "mean_tpot_ms": metrics.mean_tpot_ms,
441
- "median_tpot_ms": metrics.median_tpot_ms,
442
- "std_tpot_ms": metrics.std_tpot_ms,
443
- "p99_tpot_ms": metrics.p99_tpot_ms,
444
- "mean_itl_ms": metrics.mean_itl_ms,
445
- "median_itl_ms": metrics.median_itl_ms,
446
- "std_itl_ms": metrics.std_itl_ms,
447
- "p99_itl_ms": metrics.p99_itl_ms,
442
"input_lens": [output.prompt_len for output in outputs],
443
"output_lens": actual_output_lens,
444
"ttfts": [output.ttft for output in outputs],
@@ -452,6 +446,47 @@ async def benchmark(
446
"generated_texts": [output.generated_text for output in outputs],
447
"errors": [output.error for output in outputs],
448
}
449
+
450
+ def process_one_metric(
451
+ # E.g., "ttft"
452
+ metric_attribute_name: str,
453
+ # E.g., "TTFT"
454
+ metric_name: str,
455
+ # E.g., "Time to First Token"
456
+ metric_header: str,
457
+ ):
458
+ # This function print and add statistics of the specified
459
+ # metric.
460
+ if metric_attribute_name not in selected_percentile_metrics:
461
+ return
462
+ print("{s:{c}^{n}}".format(s=metric_header, n=50, c='-'))
463
+ print("{:<40} {:<10.2f}".format(
464
+ f"Mean {metric_name} (ms):",
465
+ getattr(metrics, f"mean_{metric_attribute_name}_ms")))
466
+ print("{:<40} {:<10.2f}".format(
467
+ f"Median {metric_name} (ms):",
468
+ getattr(metrics, f"median_{metric_attribute_name}_ms")))
469
+ result[f"mean_{metric_attribute_name}_ms"] = getattr(
470
+ metrics, f"mean_{metric_attribute_name}_ms")
471
+ result[f"median_{metric_attribute_name}_ms"] = getattr(
472
+ metrics, f"median_{metric_attribute_name}_ms")
473
+ result[f"std_{metric_attribute_name}_ms"] = getattr(
474
+ metrics, f"std_{metric_attribute_name}_ms")
475
+ for p, value in getattr(metrics,
476
+ f"percentiles_{metric_attribute_name}_ms"):
477
+ p_word = str(int(p)) if int(p) == p else str(p)
478
+ print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):",
479
+ value))
480
+ result[f"p{p_word}_{metric_attribute_name}_ms"] = value
481
+
482
+ process_one_metric("ttft", "TTFT", "Time to First Token")
483
+ process_one_metric("tpot", "TPOT",
484
+ "Time per Output Token (excl. 1st token)")
485
+ process_one_metric("itl", "ITL", "Inter-token Latency")
486
+ process_one_metric("e2el", "E2EL", "End-to-end Latency")
487
+
488
+ print("=" * 50)
489
+
490
return result
491
492
@@ -550,6 +585,10 @@ def main(args: argparse.Namespace):
585
request_rate=args.request_rate,
586
disable_tqdm=args.disable_tqdm,
587
profile=args.profile,
588
+ selected_percentile_metrics=args.percentile_metrics.split(","),
589
+ selected_percentiles=[
590
+ float(p) for p in args.metric_percentiles.split(",")
591
+ ],
592
))
593
594
# Save config and results to json
@@ -765,6 +804,23 @@ if __name__ == "__main__":
804
"{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
805
" format.",
806
)
807
+ parser.add_argument(
808
+ "--percentile-metrics",
809
+ type=str,
810
+ default="ttft,tpot,itl",
811
+ help="Comma-seperated list of selected metrics to report percentils. "
812
+ "This argument specifies the metrics to report percentiles. "
813
+ "Allowed metric names are \"ttft\", \"tpot\", \"itl\", \"e2el\". "
814
+ "Default value is \"ttft,tpot,itl\".")
815
+ parser.add_argument(
816
+ "--metric-percentiles",
817
+ type=str,
818
+ default="99",
819
+ help="Comma-seperated list of percentiles for selected metrics. "
820
+ "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "
821
+ "Default value is \"99\". "
822
+ "Use \"--percentile-metrics\" to select metrics.",
823
+ )
824
825
args = parser.parse_args()
826
main(args)