@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 855e0e6f97
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Frontend][Misc] Goodput metric support (#9338)

Andy Dai committed Oct 20, 2024 at 11:39 UTC 855e0e6f97e5ddd5addf042f25c1f11522214569
1 file changed +91 -2
benchmarks/benchmark_serving.py
+91 -2
@@ -53,6 +53,8 @@ try:
53 except ImportError:
54 from argparse import ArgumentParser as FlexibleArgumentParser
55
56 +MILLISECONDS_TO_SECONDS_CONVERSION = 1000
57 +
58
59 @dataclass
60 class BenchmarkMetrics:
@@ -60,6 +62,7 @@ class BenchmarkMetrics:
62 total_input: int
63 total_output: int
64 request_throughput: float
65 + request_goodput: float
66 output_throughput: float
67 total_token_throughput: float
68 mean_ttft_ms: float
@@ -316,12 +319,15 @@ def calculate_metrics(
319 tokenizer: PreTrainedTokenizerBase,
320 selected_percentile_metrics: List[str],
321 selected_percentiles: List[float],
322 + gootput_config_dict: Dict[str, float],
323 ) -> Tuple[BenchmarkMetrics, List[int]]:
324 actual_output_lens: List[int] = []
325 total_input = 0
326 completed = 0
327 + good_completed = 0
328 itls: List[float] = []
329 tpots: List[float] = []
330 + all_tpots: List[float] = []
331 ttfts: List[float] = []
332 e2els: List[float] = []
333 for i in range(len(outputs)):
@@ -335,9 +341,13 @@ def calculate_metrics(
341 add_special_tokens=False).input_ids)
342 actual_output_lens.append(output_len)
343 total_input += input_requests[i][1]
344 + tpot = 0
345 if output_len > 1:
339 - tpots.append(
340 - (outputs[i].latency - outputs[i].ttft) / (output_len - 1))
346 + tpot = (outputs[i].latency - outputs[i].ttft) / (output_len -
347 + 1)
348 + tpots.append(tpot)
349 + # Note: if output_len <= 1, we regard tpot as 0 for goodput
350 + all_tpots.append(tpot)
351 itls += outputs[i].itl
352 ttfts.append(outputs[i].ttft)
353 e2els.append(outputs[i].latency)
@@ -345,6 +355,28 @@ def calculate_metrics(
355 else:
356 actual_output_lens.append(0)
357
358 + if gootput_config_dict:
359 + valid_metrics = []
360 + slo_values = []
361 +
362 + if "ttft" in gootput_config_dict:
363 + valid_metrics.append(ttfts)
364 + slo_values.append(gootput_config_dict["ttft"] /
365 + MILLISECONDS_TO_SECONDS_CONVERSION)
366 + if "tpot" in gootput_config_dict:
367 + valid_metrics.append(all_tpots)
368 + slo_values.append(gootput_config_dict["tpot"] /
369 + MILLISECONDS_TO_SECONDS_CONVERSION)
370 + if "e2el" in gootput_config_dict:
371 + valid_metrics.append(e2els)
372 + slo_values.append(gootput_config_dict["e2el"] /
373 + MILLISECONDS_TO_SECONDS_CONVERSION)
374 +
375 + for req_metric in zip(*valid_metrics):
376 + is_good_req = all([s >= r for s, r in zip(slo_values, req_metric)])
377 + if is_good_req:
378 + good_completed += 1
379 +
380 if completed == 0:
381 warnings.warn(
382 "All requests failed. This is likely due to a misconfiguration "
@@ -355,6 +387,7 @@ def calculate_metrics(
387 total_input=total_input,
388 total_output=sum(actual_output_lens),
389 request_throughput=completed / dur_s,
390 + request_goodput=good_completed / dur_s,
391 output_throughput=sum(actual_output_lens) / dur_s,
392 total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s,
393 mean_ttft_ms=np.mean(ttfts or 0) *
@@ -398,6 +431,7 @@ async def benchmark(
431 selected_percentile_metrics: List[str],
432 selected_percentiles: List[str],
433 ignore_eos: bool,
434 + gootput_config_dict: Dict[str, float],
435 max_concurrency: Optional[int],
436 ):
437 if backend in ASYNC_REQUEST_FUNCS:
@@ -512,6 +546,7 @@ async def benchmark(
546 tokenizer=tokenizer,
547 selected_percentile_metrics=selected_percentile_metrics,
548 selected_percentiles=selected_percentiles,
549 + gootput_config_dict=gootput_config_dict,
550 )
551
552 print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
@@ -523,6 +558,9 @@ async def benchmark(
558 metrics.total_output))
559 print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
560 metrics.request_throughput))
561 + if gootput_config_dict:
562 + print("{:<40} {:<10.2f}".format("Request goodput (req/s):",
563 + metrics.request_goodput))
564 print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
565 metrics.output_throughput))
566 print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):",
@@ -534,6 +572,8 @@ async def benchmark(
572 "total_input_tokens": metrics.total_input,
573 "total_output_tokens": metrics.total_output,
574 "request_throughput": metrics.request_throughput,
575 + "request_goodput:":
576 + metrics.request_goodput if gootput_config_dict else None,
577 "output_throughput": metrics.output_throughput,
578 "total_token_throughput": metrics.total_token_throughput,
579 "input_lens": [output.prompt_len for output in outputs],
@@ -587,6 +627,41 @@ async def benchmark(
627 return result
628
629
630 +def check_goodput_args(args):
631 + # Check and parse goodput arguments
632 + gootput_config_dict = {}
633 + VALID_NAMES = ["ttft", "tpot", "e2el"]
634 + if args.goodput:
635 + gootput_config_dict = parse_goodput(args.goodput)
636 + for slo_name, slo_val in gootput_config_dict.items():
637 + if slo_name not in VALID_NAMES:
638 + raise ValueError(
639 + f"Invalid metric name found, {slo_name}: {slo_val}. "
640 + "The service level objective name should be one of "
641 + f"{str(VALID_NAMES)}. ")
642 + if slo_val < 0:
643 + raise ValueError(
644 + f"Invalid value found, {slo_name}: {slo_val}. "
645 + "The service level objective value should be "
646 + "non-negative.")
647 + return gootput_config_dict
648 +
649 +
650 +def parse_goodput(slo_pairs):
651 + gootput_config_dict = {}
652 + try:
653 + for slo_pair in slo_pairs:
654 + slo_name, slo_val = slo_pair.split(":")
655 + gootput_config_dict[slo_name] = float(slo_val)
656 + except ValueError as err:
657 + raise argparse.ArgumentTypeError(
658 + "Invalid format found for service level objectives. "
659 + "Specify service level objectives for goodput as \"KEY:VALUE\" "
660 + "pairs, where the key is a metric name, and the value is a "
661 + "number in milliseconds.") from err
662 + return gootput_config_dict
663 +
664 +
665 def main(args: argparse.Namespace):
666 print(args)
667 random.seed(args.seed)
@@ -681,6 +756,8 @@ def main(args: argparse.Namespace):
756 else:
757 raise ValueError(f"Unknown dataset: {args.dataset_name}")
758
759 + gootput_config_dict = check_goodput_args(args)
760 +
761 benchmark_result = asyncio.run(
762 benchmark(
763 backend=backend,
@@ -699,6 +776,7 @@ def main(args: argparse.Namespace):
776 float(p) for p in args.metric_percentiles.split(",")
777 ],
778 ignore_eos=args.ignore_eos,
779 + gootput_config_dict=gootput_config_dict,
780 max_concurrency=args.max_concurrency,
781 ))
782
@@ -915,6 +993,17 @@ if __name__ == "__main__":
993 "Default value is \"99\". "
994 "Use \"--percentile-metrics\" to select metrics.",
995 )
996 + parser.add_argument(
997 + "--goodput",
998 + nargs="+",
999 + required=False,
1000 + help="Specify service level objectives for goodput as \"KEY:VALUE\" "
1001 + "pairs, where the key is a metric name, and the value is in "
1002 + "milliseconds. Multiple \"KEY:VALUE\" pairs can be provided, "
1003 + "separated by spaces. Allowed request level metric names are "
1004 + "\"ttft\", \"tpot\", \"e2el\". For more context on the definition of "
1005 + "goodput, refer to DistServe paper: https://arxiv.org/pdf/2401.09670 "
1006 + "and the blog: https://hao-ai-lab.github.io/blogs/distserve")
1007
1008 # group for dataset specific arguments
1009 sonnet_group = parser.add_argument_group("sonnet dataset options")