@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0c785d344d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

Add more percentiles and latencies (#7759)

Wei-Sheng Chin committed Aug 29, 2024 at 16:48 UTC 0c785d344db23644139940d19d5c448754ef53d7
1 file changed +94 -38
benchmarks/benchmark_serving.py
+94 -38
@@ -61,15 +61,22 @@ class BenchmarkMetrics:
61 mean_ttft_ms: float
62 median_ttft_ms: float
63 std_ttft_ms: float
64 - p99_ttft_ms: float
64 + percentiles_ttft_ms: List[Tuple[float, float]]
65 mean_tpot_ms: float
66 median_tpot_ms: float
67 std_tpot_ms: float
68 - p99_tpot_ms: float
68 + percentiles_tpot_ms: List[Tuple[float, float]]
69 mean_itl_ms: float
70 median_itl_ms: float
71 std_itl_ms: float
72 - p99_itl_ms: float
72 + percentiles_itl_ms: List[Tuple[float, float]]
73 + # E2EL stands for end-to-end latency per request.
74 + # It is the time taken on the client side from sending
75 + # a request to receiving a complete response.
76 + mean_e2el_ms: float
77 + median_e2el_ms: float
78 + std_e2el_ms: float
79 + percentiles_e2el_ms: List[Tuple[float, float]]
80
81
82 def sample_sharegpt_requests(
@@ -235,6 +242,8 @@ def calculate_metrics(
242 outputs: List[RequestFuncOutput],
243 dur_s: float,
244 tokenizer: PreTrainedTokenizerBase,
245 + selected_percentile_metrics: List[str],
246 + selected_percentiles: List[float],
247 ) -> Tuple[BenchmarkMetrics, List[int]]:
248 actual_output_lens: List[int] = []
249 total_input = 0
@@ -242,6 +251,7 @@ def calculate_metrics(
251 itls: List[float] = []
252 tpots: List[float] = []
253 ttfts: List[float] = []
254 + e2els: List[float] = []
255 for i in range(len(outputs)):
256 if outputs[i].success:
257 # We use the tokenizer to count the number of output tokens for all
@@ -258,6 +268,7 @@ def calculate_metrics(
268 (outputs[i].latency - outputs[i].ttft) / (output_len - 1))
269 itls += outputs[i].itl
270 ttfts.append(outputs[i].ttft)
271 + e2els.append(outputs[i].latency)
272 completed += 1
273 else:
274 actual_output_lens.append(0)
@@ -276,17 +287,25 @@ def calculate_metrics(
287 output_throughput=sum(actual_output_lens) / dur_s,
288 mean_ttft_ms=np.mean(ttfts or 0) *
289 1000, # ttfts is empty if streaming is not supported by backend
279 - median_ttft_ms=np.median(ttfts or 0) * 1000,
290 std_ttft_ms=np.std(ttfts or 0) * 1000,
281 - p99_ttft_ms=np.percentile(ttfts or 0, 99) * 1000,
291 + median_ttft_ms=np.median(ttfts or 0) * 1000,
292 + percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000)
293 + for p in selected_percentiles],
294 mean_tpot_ms=np.mean(tpots or 0) * 1000,
283 - median_tpot_ms=np.median(tpots or 0) * 1000,
295 std_tpot_ms=np.std(tpots or 0) * 1000,
285 - p99_tpot_ms=np.percentile(tpots or 0, 99) * 1000,
296 + median_tpot_ms=np.median(tpots or 0) * 1000,
297 + percentiles_tpot_ms=[(p, np.percentile(tpots or 0, p) * 1000)
298 + for p in selected_percentiles],
299 mean_itl_ms=np.mean(itls or 0) * 1000,
287 - median_itl_ms=np.median(itls or 0) * 1000,
300 std_itl_ms=np.std(itls or 0) * 1000,
289 - p99_itl_ms=np.percentile(itls or 0, 99) * 1000,
301 + median_itl_ms=np.median(itls or 0) * 1000,
302 + percentiles_itl_ms=[(p, np.percentile(itls or 0, p) * 1000)
303 + for p in selected_percentiles],
304 + mean_e2el_ms=np.median(e2els or 0) * 1000,
305 + std_e2el_ms=np.std(e2els or 0) * 1000,
306 + median_e2el_ms=np.mean(e2els or 0) * 1000,
307 + percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000)
308 + for p in selected_percentiles],
309 )
310
311 return metrics, actual_output_lens
@@ -304,6 +323,8 @@ async def benchmark(
323 request_rate: float,
324 disable_tqdm: bool,
325 profile: bool,
326 + selected_percentile_metrics: List[str],
327 + selected_percentiles: List[str],
328 ):
329 if backend in ASYNC_REQUEST_FUNCS:
330 request_func = ASYNC_REQUEST_FUNCS[backend]
@@ -392,6 +413,8 @@ async def benchmark(
413 outputs=outputs,
414 dur_s=benchmark_duration,
415 tokenizer=tokenizer,
416 + selected_percentile_metrics=selected_percentile_metrics,
417 + selected_percentiles=selected_percentiles,
418 )
419
420 print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
@@ -407,23 +430,6 @@ async def benchmark(
430 metrics.input_throughput))
431 print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
432 metrics.output_throughput))
410 - print("{s:{c}^{n}}".format(s='Time to First Token', n=50, c='-'))
411 - print("{:<40} {:<10.2f}".format("Mean TTFT (ms):", metrics.mean_ttft_ms))
412 - print("{:<40} {:<10.2f}".format("Median TTFT (ms):",
413 - metrics.median_ttft_ms))
414 - print("{:<40} {:<10.2f}".format("P99 TTFT (ms):", metrics.p99_ttft_ms))
415 - print("{s:{c}^{n}}".format(s='Time per Output Token (excl. 1st token)',
416 - n=50,
417 - c='-'))
418 - print("{:<40} {:<10.2f}".format("Mean TPOT (ms):", metrics.mean_tpot_ms))
419 - print("{:<40} {:<10.2f}".format("Median TPOT (ms):",
420 - metrics.median_tpot_ms))
421 - print("{:<40} {:<10.2f}".format("P99 TPOT (ms):", metrics.p99_tpot_ms))
422 - print("{s:{c}^{n}}".format(s='Inter-token Latency', n=50, c='-'))
423 - print("{:<40} {:<10.2f}".format("Mean ITL (ms):", metrics.mean_itl_ms))
424 - print("{:<40} {:<10.2f}".format("Median ITL (ms):", metrics.median_itl_ms))
425 - print("{:<40} {:<10.2f}".format("P99 ITL (ms):", metrics.p99_itl_ms))
426 - print("=" * 50)
433
434 result = {
435 "duration": benchmark_duration,
@@ -433,18 +439,6 @@ async def benchmark(
439 "request_throughput": metrics.request_throughput,
440 "input_throughput": metrics.input_throughput,
441 "output_throughput": metrics.output_throughput,
436 - "mean_ttft_ms": metrics.mean_ttft_ms,
437 - "median_ttft_ms": metrics.median_ttft_ms,
438 - "std_ttft_ms": metrics.std_ttft_ms,
439 - "p99_ttft_ms": metrics.p99_ttft_ms,
440 - "mean_tpot_ms": metrics.mean_tpot_ms,
441 - "median_tpot_ms": metrics.median_tpot_ms,
442 - "std_tpot_ms": metrics.std_tpot_ms,
443 - "p99_tpot_ms": metrics.p99_tpot_ms,
444 - "mean_itl_ms": metrics.mean_itl_ms,
445 - "median_itl_ms": metrics.median_itl_ms,
446 - "std_itl_ms": metrics.std_itl_ms,
447 - "p99_itl_ms": metrics.p99_itl_ms,
442 "input_lens": [output.prompt_len for output in outputs],
443 "output_lens": actual_output_lens,
444 "ttfts": [output.ttft for output in outputs],
@@ -452,6 +446,47 @@ async def benchmark(
446 "generated_texts": [output.generated_text for output in outputs],
447 "errors": [output.error for output in outputs],
448 }
449 +
450 + def process_one_metric(
451 + # E.g., "ttft"
452 + metric_attribute_name: str,
453 + # E.g., "TTFT"
454 + metric_name: str,
455 + # E.g., "Time to First Token"
456 + metric_header: str,
457 + ):
458 + # This function print and add statistics of the specified
459 + # metric.
460 + if metric_attribute_name not in selected_percentile_metrics:
461 + return
462 + print("{s:{c}^{n}}".format(s=metric_header, n=50, c='-'))
463 + print("{:<40} {:<10.2f}".format(
464 + f"Mean {metric_name} (ms):",
465 + getattr(metrics, f"mean_{metric_attribute_name}_ms")))
466 + print("{:<40} {:<10.2f}".format(
467 + f"Median {metric_name} (ms):",
468 + getattr(metrics, f"median_{metric_attribute_name}_ms")))
469 + result[f"mean_{metric_attribute_name}_ms"] = getattr(
470 + metrics, f"mean_{metric_attribute_name}_ms")
471 + result[f"median_{metric_attribute_name}_ms"] = getattr(
472 + metrics, f"median_{metric_attribute_name}_ms")
473 + result[f"std_{metric_attribute_name}_ms"] = getattr(
474 + metrics, f"std_{metric_attribute_name}_ms")
475 + for p, value in getattr(metrics,
476 + f"percentiles_{metric_attribute_name}_ms"):
477 + p_word = str(int(p)) if int(p) == p else str(p)
478 + print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):",
479 + value))
480 + result[f"p{p_word}_{metric_attribute_name}_ms"] = value
481 +
482 + process_one_metric("ttft", "TTFT", "Time to First Token")
483 + process_one_metric("tpot", "TPOT",
484 + "Time per Output Token (excl. 1st token)")
485 + process_one_metric("itl", "ITL", "Inter-token Latency")
486 + process_one_metric("e2el", "E2EL", "End-to-end Latency")
487 +
488 + print("=" * 50)
489 +
490 return result
491
492
@@ -550,6 +585,10 @@ def main(args: argparse.Namespace):
585 request_rate=args.request_rate,
586 disable_tqdm=args.disable_tqdm,
587 profile=args.profile,
588 + selected_percentile_metrics=args.percentile_metrics.split(","),
589 + selected_percentiles=[
590 + float(p) for p in args.metric_percentiles.split(",")
591 + ],
592 ))
593
594 # Save config and results to json
@@ -765,6 +804,23 @@ if __name__ == "__main__":
804 "{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
805 " format.",
806 )
807 + parser.add_argument(
808 + "--percentile-metrics",
809 + type=str,
810 + default="ttft,tpot,itl",
811 + help="Comma-seperated list of selected metrics to report percentils. "
812 + "This argument specifies the metrics to report percentiles. "
813 + "Allowed metric names are \"ttft\", \"tpot\", \"itl\", \"e2el\". "
814 + "Default value is \"ttft,tpot,itl\".")
815 + parser.add_argument(
816 + "--metric-percentiles",
817 + type=str,
818 + default="99",
819 + help="Comma-seperated list of percentiles for selected metrics. "
820 + "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "
821 + "Default value is \"99\". "
822 + "Use \"--percentile-metrics\" to select metrics.",
823 + )
824
825 args = parser.parse_args()
826 main(args)