A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Frontend][Misc] Goodput metric support (#9338)
Andy Dai committed
Oct 20, 2024 at 11:39 UTC
855e0e6f97e5ddd5addf042f25c1f11522214569
1 file changed
+91
-2
benchmarks/benchmark_serving.py
+91
-2
@@ -53,6 +53,8 @@ try:
53
except ImportError:
54
from argparse import ArgumentParser as FlexibleArgumentParser
55
56
+MILLISECONDS_TO_SECONDS_CONVERSION = 1000
57
+
58
59
@dataclass
60
class BenchmarkMetrics:
@@ -60,6 +62,7 @@ class BenchmarkMetrics:
62
total_input: int
63
total_output: int
64
request_throughput: float
65
+ request_goodput: float
66
output_throughput: float
67
total_token_throughput: float
68
mean_ttft_ms: float
@@ -316,12 +319,15 @@ def calculate_metrics(
319
tokenizer: PreTrainedTokenizerBase,
320
selected_percentile_metrics: List[str],
321
selected_percentiles: List[float],
322
+ gootput_config_dict: Dict[str, float],
323
) -> Tuple[BenchmarkMetrics, List[int]]:
324
actual_output_lens: List[int] = []
325
total_input = 0
326
completed = 0
327
+ good_completed = 0
328
itls: List[float] = []
329
tpots: List[float] = []
330
+ all_tpots: List[float] = []
331
ttfts: List[float] = []
332
e2els: List[float] = []
333
for i in range(len(outputs)):
@@ -335,9 +341,13 @@ def calculate_metrics(
341
add_special_tokens=False).input_ids)
342
actual_output_lens.append(output_len)
343
total_input += input_requests[i][1]
344
+ tpot = 0
345
if output_len > 1:
339
- tpots.append(
340
- (outputs[i].latency - outputs[i].ttft) / (output_len - 1))
346
+ tpot = (outputs[i].latency - outputs[i].ttft) / (output_len -
347
+ 1)
348
+ tpots.append(tpot)
349
+ # Note: if output_len <= 1, we regard tpot as 0 for goodput
350
+ all_tpots.append(tpot)
351
itls += outputs[i].itl
352
ttfts.append(outputs[i].ttft)
353
e2els.append(outputs[i].latency)
@@ -345,6 +355,28 @@ def calculate_metrics(
355
else:
356
actual_output_lens.append(0)
357
358
+ if gootput_config_dict:
359
+ valid_metrics = []
360
+ slo_values = []
361
+
362
+ if "ttft" in gootput_config_dict:
363
+ valid_metrics.append(ttfts)
364
+ slo_values.append(gootput_config_dict["ttft"] /
365
+ MILLISECONDS_TO_SECONDS_CONVERSION)
366
+ if "tpot" in gootput_config_dict:
367
+ valid_metrics.append(all_tpots)
368
+ slo_values.append(gootput_config_dict["tpot"] /
369
+ MILLISECONDS_TO_SECONDS_CONVERSION)
370
+ if "e2el" in gootput_config_dict:
371
+ valid_metrics.append(e2els)
372
+ slo_values.append(gootput_config_dict["e2el"] /
373
+ MILLISECONDS_TO_SECONDS_CONVERSION)
374
+
375
+ for req_metric in zip(*valid_metrics):
376
+ is_good_req = all([s >= r for s, r in zip(slo_values, req_metric)])
377
+ if is_good_req:
378
+ good_completed += 1
379
+
380
if completed == 0:
381
warnings.warn(
382
"All requests failed. This is likely due to a misconfiguration "
@@ -355,6 +387,7 @@ def calculate_metrics(
387
total_input=total_input,
388
total_output=sum(actual_output_lens),
389
request_throughput=completed / dur_s,
390
+ request_goodput=good_completed / dur_s,
391
output_throughput=sum(actual_output_lens) / dur_s,
392
total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s,
393
mean_ttft_ms=np.mean(ttfts or 0) *
@@ -398,6 +431,7 @@ async def benchmark(
431
selected_percentile_metrics: List[str],
432
selected_percentiles: List[str],
433
ignore_eos: bool,
434
+ gootput_config_dict: Dict[str, float],
435
max_concurrency: Optional[int],
436
):
437
if backend in ASYNC_REQUEST_FUNCS:
@@ -512,6 +546,7 @@ async def benchmark(
546
tokenizer=tokenizer,
547
selected_percentile_metrics=selected_percentile_metrics,
548
selected_percentiles=selected_percentiles,
549
+ gootput_config_dict=gootput_config_dict,
550
)
551
552
print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
@@ -523,6 +558,9 @@ async def benchmark(
558
metrics.total_output))
559
print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
560
metrics.request_throughput))
561
+ if gootput_config_dict:
562
+ print("{:<40} {:<10.2f}".format("Request goodput (req/s):",
563
+ metrics.request_goodput))
564
print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
565
metrics.output_throughput))
566
print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):",
@@ -534,6 +572,8 @@ async def benchmark(
572
"total_input_tokens": metrics.total_input,
573
"total_output_tokens": metrics.total_output,
574
"request_throughput": metrics.request_throughput,
575
+ "request_goodput:":
576
+ metrics.request_goodput if gootput_config_dict else None,
577
"output_throughput": metrics.output_throughput,
578
"total_token_throughput": metrics.total_token_throughput,
579
"input_lens": [output.prompt_len for output in outputs],
@@ -587,6 +627,41 @@ async def benchmark(
627
return result
628
629
630
+def check_goodput_args(args):
631
+ # Check and parse goodput arguments
632
+ gootput_config_dict = {}
633
+ VALID_NAMES = ["ttft", "tpot", "e2el"]
634
+ if args.goodput:
635
+ gootput_config_dict = parse_goodput(args.goodput)
636
+ for slo_name, slo_val in gootput_config_dict.items():
637
+ if slo_name not in VALID_NAMES:
638
+ raise ValueError(
639
+ f"Invalid metric name found, {slo_name}: {slo_val}. "
640
+ "The service level objective name should be one of "
641
+ f"{str(VALID_NAMES)}. ")
642
+ if slo_val < 0:
643
+ raise ValueError(
644
+ f"Invalid value found, {slo_name}: {slo_val}. "
645
+ "The service level objective value should be "
646
+ "non-negative.")
647
+ return gootput_config_dict
648
+
649
+
650
+def parse_goodput(slo_pairs):
651
+ gootput_config_dict = {}
652
+ try:
653
+ for slo_pair in slo_pairs:
654
+ slo_name, slo_val = slo_pair.split(":")
655
+ gootput_config_dict[slo_name] = float(slo_val)
656
+ except ValueError as err:
657
+ raise argparse.ArgumentTypeError(
658
+ "Invalid format found for service level objectives. "
659
+ "Specify service level objectives for goodput as \"KEY:VALUE\" "
660
+ "pairs, where the key is a metric name, and the value is a "
661
+ "number in milliseconds.") from err
662
+ return gootput_config_dict
663
+
664
+
665
def main(args: argparse.Namespace):
666
print(args)
667
random.seed(args.seed)
@@ -681,6 +756,8 @@ def main(args: argparse.Namespace):
756
else:
757
raise ValueError(f"Unknown dataset: {args.dataset_name}")
758
759
+ gootput_config_dict = check_goodput_args(args)
760
+
761
benchmark_result = asyncio.run(
762
benchmark(
763
backend=backend,
@@ -699,6 +776,7 @@ def main(args: argparse.Namespace):
776
float(p) for p in args.metric_percentiles.split(",")
777
],
778
ignore_eos=args.ignore_eos,
779
+ gootput_config_dict=gootput_config_dict,
780
max_concurrency=args.max_concurrency,
781
))
782
@@ -915,6 +993,17 @@ if __name__ == "__main__":
993
"Default value is \"99\". "
994
"Use \"--percentile-metrics\" to select metrics.",
995
)
996
+ parser.add_argument(
997
+ "--goodput",
998
+ nargs="+",
999
+ required=False,
1000
+ help="Specify service level objectives for goodput as \"KEY:VALUE\" "
1001
+ "pairs, where the key is a metric name, and the value is in "
1002
+ "milliseconds. Multiple \"KEY:VALUE\" pairs can be provided, "
1003
+ "separated by spaces. Allowed request level metric names are "
1004
+ "\"ttft\", \"tpot\", \"e2el\". For more context on the definition of "
1005
+ "goodput, refer to DistServe paper: https://arxiv.org/pdf/2401.09670 "
1006
+ "and the blog: https://hao-ai-lab.github.io/blogs/distserve")
1007
1008
# group for dataset specific arguments
1009
sonnet_group = parser.add_argument_group("sonnet dataset options")