A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Misc]: Add support for goodput on guided benchmarking + TPOT calculation refactor (#13736)
Signed-off-by: Brayden Zhong <b8zhong@uwaterloo.ca>
Brayden Zhong committed
Feb 26, 2025 at 06:06 UTC
ec8a5e53863826eb25a9b242535d4671bbb3f6ce
1 file changed
+82
-5
benchmarks/benchmark_serving_guided.py
+82
-5
@@ -9,7 +9,7 @@ On the server side, run one of the following commands:
9
./launch_tgi_server.sh <your_model> <max_batch_total_tokens>
10
11
On the client side, run:
12
- python benchmarks/benchmark_serving.py \
12
+ python benchmarks/benchmark_serving_guided.py \
13
--backend <backend> \
14
--model <your_model> \
15
--dataset json \
@@ -31,7 +31,7 @@ import random
31
import time
32
import warnings
33
from dataclasses import dataclass
34
-from typing import AsyncGenerator, List, Optional, Tuple
34
+from typing import AsyncGenerator, Dict, List, Optional, Tuple
35
36
import datasets
37
import numpy as np
@@ -264,6 +264,7 @@ def calculate_metrics(
264
tokenizer: PreTrainedTokenizerBase,
265
selected_percentile_metrics: List[str],
266
selected_percentiles: List[float],
267
+ goodput_config_dict: Optional[Dict[str, float]] = None,
268
) -> Tuple[BenchmarkMetrics, List[int]]:
269
actual_output_lens: List[int] = []
270
total_input = 0
@@ -287,10 +288,10 @@ def calculate_metrics(
288
total_input += input_requests[i].prompt_len
289
tpot = 0
290
if output_len > 1:
290
- tpot = (outputs[i].latency - outputs[i].ttft) / (output_len -
291
- 1)
291
+ latency_minus_ttft = outputs[i].latency - outputs[i].ttft
292
+ tpot = latency_minus_ttft / (output_len - 1)
293
tpots.append(tpot)
293
- outputs[i].tpot = sum(tpots) / len(tpots) if len(tpots) else 0
294
+ outputs[i].tpot = tpot
295
# Note: if output_len <= 1, we regard tpot as 0 for goodput
296
all_tpots.append(tpot)
297
itls += outputs[i].itl
@@ -300,6 +301,28 @@ def calculate_metrics(
301
else:
302
actual_output_lens.append(0)
303
304
+ if goodput_config_dict:
305
+ valid_metrics = []
306
+ slo_values = []
307
+
308
+ if "ttft" in goodput_config_dict:
309
+ valid_metrics.append(ttfts)
310
+ slo_values.append(goodput_config_dict["ttft"] /
311
+ MILLISECONDS_TO_SECONDS_CONVERSION)
312
+ if "tpot" in goodput_config_dict:
313
+ valid_metrics.append(all_tpots)
314
+ slo_values.append(goodput_config_dict["tpot"] /
315
+ MILLISECONDS_TO_SECONDS_CONVERSION)
316
+ if "e2el" in goodput_config_dict:
317
+ valid_metrics.append(e2els)
318
+ slo_values.append(goodput_config_dict["e2el"] /
319
+ MILLISECONDS_TO_SECONDS_CONVERSION)
320
+
321
+ for req_metric in zip(*valid_metrics):
322
+ is_good_req = all([s >= r for s, r in zip(slo_values, req_metric)])
323
+ if is_good_req:
324
+ good_completed += 1
325
+
326
if completed == 0:
327
warnings.warn(
328
"All requests failed. This is likely due to a misconfiguration "
@@ -356,6 +379,7 @@ async def benchmark(
379
max_concurrency: Optional[int],
380
guided_decoding_ratio: float,
381
guided_decoding_backend: str,
382
+ goodput_config_dict: Optional[Dict[str, float]] = None,
383
):
384
if backend in ASYNC_REQUEST_FUNCS:
385
request_func = ASYNC_REQUEST_FUNCS[backend]
@@ -483,6 +507,7 @@ async def benchmark(
507
tokenizer=tokenizer,
508
selected_percentile_metrics=selected_percentile_metrics,
509
selected_percentiles=selected_percentiles,
510
+ goodput_config_dict=goodput_config_dict,
511
)
512
513
print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
@@ -494,6 +519,9 @@ async def benchmark(
519
metrics.total_output))
520
print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
521
metrics.request_throughput))
522
+ if goodput_config_dict:
523
+ print("{:<40} {:<10.2f}".format("Request goodput (req/s):",
524
+ metrics.request_goodput))
525
print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
526
metrics.output_throughput))
527
print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):",
@@ -617,6 +645,40 @@ def evaluate(ret, args):
645
100) if len(not_none_scores) > 0 else None
646
647
648
+def parse_goodput(slo_pairs):
649
+ goodput_config_dict = {}
650
+ try:
651
+ for slo_pair in slo_pairs:
652
+ slo_name, slo_val = slo_pair.split(":")
653
+ goodput_config_dict[slo_name] = float(slo_val)
654
+ except ValueError as err:
655
+ raise argparse.ArgumentTypeError(
656
+ "Invalid format found for service level objectives. "
657
+ "Specify service level objectives for goodput as \"KEY:VALUE\" "
658
+ "pairs, where the key is a metric name, and the value is a "
659
+ "number in milliseconds.") from err
660
+ return goodput_config_dict
661
+
662
+
663
+def check_goodput_args(args):
664
+ goodput_config_dict = {}
665
+ VALID_NAMES = ["ttft", "tpot", "e2el"]
666
+ if args.goodput:
667
+ goodput_config_dict = parse_goodput(args.goodput)
668
+ for slo_name, slo_val in goodput_config_dict.items():
669
+ if slo_name not in VALID_NAMES:
670
+ raise ValueError(
671
+ f"Invalid metric name found, {slo_name}: {slo_val}. "
672
+ "The service level objective name should be one of "
673
+ f"{str(VALID_NAMES)}. ")
674
+ if slo_val < 0:
675
+ raise ValueError(
676
+ f"Invalid value found, {slo_name}: {slo_val}. "
677
+ "The service level objective value should be "
678
+ "non-negative.")
679
+ return goodput_config_dict
680
+
681
+
682
def main(args: argparse.Namespace):
683
print(args)
684
random.seed(args.seed)
@@ -661,6 +723,8 @@ def main(args: argparse.Namespace):
723
724
input_requests = sample_requests(tokenizer, args)
725
726
+ goodput_config_dict = check_goodput_args(args)
727
+
728
benchmark_result, ret = asyncio.run(
729
benchmark(
730
backend=backend,
@@ -681,6 +745,7 @@ def main(args: argparse.Namespace):
745
max_concurrency=args.max_concurrency,
746
guided_decoding_ratio=args.guided_decoding_ratio,
747
guided_decoding_backend=args.guided_decoding_backend,
748
+ goodput_config_dict=goodput_config_dict,
749
))
750
751
# Save config and results to json
@@ -865,6 +930,18 @@ if __name__ == "__main__":
930
"Default value is \"99\". "
931
"Use \"--percentile-metrics\" to select metrics.",
932
)
933
+ parser.add_argument(
934
+ "--goodput",
935
+ nargs="+",
936
+ required=False,
937
+ help="Specify service level objectives for goodput as \"KEY:VALUE\" "
938
+ "pairs, where the key is a metric name, and the value is in "
939
+ "milliseconds. Multiple \"KEY:VALUE\" pairs can be provided, "
940
+ "separated by spaces. Allowed request level metric names are "
941
+ "\"ttft\", \"tpot\", \"e2el\". For more context on the definition of "
942
+ "goodput, refer to DistServe paper: https://arxiv.org/pdf/2401.09670 "
943
+ "and the blog: https://hao-ai-lab.github.io/blogs/distserve")
944
+
945
parser.add_argument("--no-guided-decoding",
946
action='store_true',
947
default=False,