@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 222a9dc350
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Benchmark] More accurate TPOT calc in `benchmark_serving.py` (#12288)

Signed-off-by: Nick Hill <nhill@redhat.com>

Nick Hill committed Jan 21, 2025 at 21:46 UTC 222a9dc350d99e111ba09b29576113c9278e6a3e
2 files changed +66 -46
benchmarks/backend_request_func.py
+27 -16
@@ -35,6 +35,7 @@ class RequestFuncOutput:
35 generated_text: str = ""
36 success: bool = False
37 latency: float = 0.0
38 + output_tokens: int = 0
39 ttft: float = 0.0 # Time to first token
40 itl: List[float] = field(
41 default_factory=list) # List of inter-token latencies
@@ -156,7 +157,7 @@ async def async_request_trt_llm(
157 timestamp = time.perf_counter()
158 # First token
159 if ttft == 0.0:
159 - ttft = time.perf_counter() - st
160 + ttft = timestamp - st
161 output.ttft = ttft
162
163 # Decoding phase
@@ -245,6 +246,9 @@ async def async_request_openai_completions(
246 "logprobs": request_func_input.logprobs,
247 "stream": True,
248 "ignore_eos": request_func_input.ignore_eos,
249 + "stream_options": {
250 + "include_usage": True,
251 + },
252 }
253 if request_func_input.extra_body:
254 payload.update(request_func_input.extra_body)
@@ -256,7 +260,6 @@ async def async_request_openai_completions(
260 output.prompt_len = request_func_input.prompt_len
261
262 generated_text = ""
259 - ttft = 0.0
263 st = time.perf_counter()
264 most_recent_timestamp = st
265 try:
@@ -271,15 +274,16 @@ async def async_request_openai_completions(
274
275 chunk = chunk_bytes.decode("utf-8").removeprefix(
276 "data: ")
274 - if chunk == "[DONE]":
275 - latency = time.perf_counter() - st
276 - else:
277 + if chunk != "[DONE]":
278 data = json.loads(chunk)
279
280 # NOTE: Some completion API might have a last
281 # usage summary response without a token so we
282 # want to check a token was generated
282 - if data["choices"][0]["text"]:
283 + if choices := data.get("choices"):
284 + # Note that text could be empty here
285 + # e.g. for special tokens
286 + text = choices[0].get("text")
287 timestamp = time.perf_counter()
288 # First token
289 if not first_chunk_received:
@@ -293,7 +297,10 @@ async def async_request_openai_completions(
297 most_recent_timestamp)
298
299 most_recent_timestamp = timestamp
296 - generated_text += data["choices"][0]["text"]
300 + generated_text += text
301 + elif usage := data.get("usage"):
302 + output.output_tokens = usage.get(
303 + "completion_tokens")
304 if first_chunk_received:
305 output.success = True
306 else:
@@ -302,7 +309,7 @@ async def async_request_openai_completions(
309 "Never received a valid chunk to calculate TTFT."
310 "This response will be marked as failed!")
311 output.generated_text = generated_text
305 - output.latency = latency
312 + output.latency = most_recent_timestamp - st
313 else:
314 output.error = response.reason or ""
315 output.success = False
@@ -342,6 +349,9 @@ async def async_request_openai_chat_completions(
349 "max_completion_tokens": request_func_input.output_len,
350 "stream": True,
351 "ignore_eos": request_func_input.ignore_eos,
352 + "stream_options": {
353 + "include_usage": True,
354 + },
355 }
356 if request_func_input.extra_body:
357 payload.update(request_func_input.extra_body)
@@ -368,17 +378,15 @@ async def async_request_openai_chat_completions(
378
379 chunk = chunk_bytes.decode("utf-8").removeprefix(
380 "data: ")
371 - if chunk == "[DONE]":
372 - latency = time.perf_counter() - st
373 - else:
381 + if chunk != "[DONE]":
382 timestamp = time.perf_counter()
383 data = json.loads(chunk)
384
377 - delta = data["choices"][0]["delta"]
378 - if delta.get("content", None):
385 + if choices := data.get("choices"):
386 + content = choices[0]["delta"].get("content")
387 # First token
388 if ttft == 0.0:
381 - ttft = time.perf_counter() - st
389 + ttft = timestamp - st
390 output.ttft = ttft
391
392 # Decoding phase
@@ -386,13 +394,16 @@ async def async_request_openai_chat_completions(
394 output.itl.append(timestamp -
395 most_recent_timestamp)
396
389 - generated_text += delta["content"]
397 + generated_text += content
398 + elif usage := data.get("usage"):
399 + output.output_tokens = usage.get(
400 + "completion_tokens")
401
402 most_recent_timestamp = timestamp
403
404 output.generated_text = generated_text
405 output.success = True
395 - output.latency = latency
406 + output.latency = most_recent_timestamp - st
407 else:
408 output.error = response.reason or ""
409 output.success = False
benchmarks/benchmark_serving.py
+39 -30
@@ -25,6 +25,7 @@ On the client side, run:
25 import argparse
26 import asyncio
27 import base64
28 +import gc
29 import io
30 import json
31 import os
@@ -423,7 +424,7 @@ def calculate_metrics(
424 tokenizer: PreTrainedTokenizerBase,
425 selected_percentile_metrics: List[str],
426 selected_percentiles: List[float],
426 - gootput_config_dict: Dict[str, float],
427 + goodput_config_dict: Dict[str, float],
428 ) -> Tuple[BenchmarkMetrics, List[int]]:
429 actual_output_lens: List[int] = []
430 total_input = 0
@@ -436,19 +437,23 @@ def calculate_metrics(
437 e2els: List[float] = []
438 for i in range(len(outputs)):
439 if outputs[i].success:
439 - # We use the tokenizer to count the number of output tokens for all
440 - # serving backends instead of looking at len(outputs[i].itl) since
441 - # multiple output tokens may be bundled together
442 - # Note : this may inflate the output token count slightly
443 - output_len = len(
444 - tokenizer(outputs[i].generated_text,
445 - add_special_tokens=False).input_ids)
440 + output_len = outputs[i].output_tokens
441 +
442 + if output_len is None:
443 + # We use the tokenizer to count the number of output tokens
444 + # for some serving backends instead of looking at
445 + # len(outputs[i].itl) since multiple output tokens may be
446 + # bundled together
447 + # Note : this may inflate the output token count slightly
448 + output_len = len(
449 + tokenizer(outputs[i].generated_text,
450 + add_special_tokens=False).input_ids)
451 actual_output_lens.append(output_len)
452 total_input += input_requests[i][1]
453 tpot = 0
454 if output_len > 1:
450 - tpot = (outputs[i].latency - outputs[i].ttft) / (output_len -
451 - 1)
455 + latency_minus_ttft = outputs[i].latency - outputs[i].ttft
456 + tpot = latency_minus_ttft / (output_len - 1)
457 tpots.append(tpot)
458 # Note: if output_len <= 1, we regard tpot as 0 for goodput
459 all_tpots.append(tpot)
@@ -459,21 +464,21 @@ def calculate_metrics(
464 else:
465 actual_output_lens.append(0)
466
462 - if gootput_config_dict:
467 + if goodput_config_dict:
468 valid_metrics = []
469 slo_values = []
470
466 - if "ttft" in gootput_config_dict:
471 + if "ttft" in goodput_config_dict:
472 valid_metrics.append(ttfts)
468 - slo_values.append(gootput_config_dict["ttft"] /
473 + slo_values.append(goodput_config_dict["ttft"] /
474 MILLISECONDS_TO_SECONDS_CONVERSION)
470 - if "tpot" in gootput_config_dict:
475 + if "tpot" in goodput_config_dict:
476 valid_metrics.append(all_tpots)
472 - slo_values.append(gootput_config_dict["tpot"] /
477 + slo_values.append(goodput_config_dict["tpot"] /
478 MILLISECONDS_TO_SECONDS_CONVERSION)
474 - if "e2el" in gootput_config_dict:
479 + if "e2el" in goodput_config_dict:
480 valid_metrics.append(e2els)
476 - slo_values.append(gootput_config_dict["e2el"] /
481 + slo_values.append(goodput_config_dict["e2el"] /
482 MILLISECONDS_TO_SECONDS_CONVERSION)
483
484 for req_metric in zip(*valid_metrics):
@@ -537,7 +542,7 @@ async def benchmark(
542 selected_percentile_metrics: List[str],
543 selected_percentiles: List[str],
544 ignore_eos: bool,
540 - gootput_config_dict: Dict[str, float],
545 + goodput_config_dict: Dict[str, float],
546 max_concurrency: Optional[int],
547 ):
548 if backend in ASYNC_REQUEST_FUNCS:
@@ -661,7 +666,7 @@ async def benchmark(
666 tokenizer=tokenizer,
667 selected_percentile_metrics=selected_percentile_metrics,
668 selected_percentiles=selected_percentiles,
664 - gootput_config_dict=gootput_config_dict,
669 + goodput_config_dict=goodput_config_dict,
670 )
671
672 print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
@@ -673,7 +678,7 @@ async def benchmark(
678 metrics.total_output))
679 print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
680 metrics.request_throughput))
676 - if gootput_config_dict:
681 + if goodput_config_dict:
682 print("{:<40} {:<10.2f}".format("Request goodput (req/s):",
683 metrics.request_goodput))
684 print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
@@ -688,7 +693,7 @@ async def benchmark(
693 "total_output_tokens": metrics.total_output,
694 "request_throughput": metrics.request_throughput,
695 "request_goodput:":
691 - metrics.request_goodput if gootput_config_dict else None,
696 + metrics.request_goodput if goodput_config_dict else None,
697 "output_throughput": metrics.output_throughput,
698 "total_token_throughput": metrics.total_token_throughput,
699 "input_lens": [output.prompt_len for output in outputs],
@@ -744,11 +749,11 @@ async def benchmark(
749
750 def check_goodput_args(args):
751 # Check and parse goodput arguments
747 - gootput_config_dict = {}
752 + goodput_config_dict = {}
753 VALID_NAMES = ["ttft", "tpot", "e2el"]
754 if args.goodput:
750 - gootput_config_dict = parse_goodput(args.goodput)
751 - for slo_name, slo_val in gootput_config_dict.items():
755 + goodput_config_dict = parse_goodput(args.goodput)
756 + for slo_name, slo_val in goodput_config_dict.items():
757 if slo_name not in VALID_NAMES:
758 raise ValueError(
759 f"Invalid metric name found, {slo_name}: {slo_val}. "
@@ -759,22 +764,22 @@ def check_goodput_args(args):
764 f"Invalid value found, {slo_name}: {slo_val}. "
765 "The service level objective value should be "
766 "non-negative.")
762 - return gootput_config_dict
767 + return goodput_config_dict
768
769
770 def parse_goodput(slo_pairs):
766 - gootput_config_dict = {}
771 + goodput_config_dict = {}
772 try:
773 for slo_pair in slo_pairs:
774 slo_name, slo_val = slo_pair.split(":")
770 - gootput_config_dict[slo_name] = float(slo_val)
775 + goodput_config_dict[slo_name] = float(slo_val)
776 except ValueError as err:
777 raise argparse.ArgumentTypeError(
778 "Invalid format found for service level objectives. "
779 "Specify service level objectives for goodput as \"KEY:VALUE\" "
780 "pairs, where the key is a metric name, and the value is a "
781 "number in milliseconds.") from err
777 - return gootput_config_dict
782 + return goodput_config_dict
783
784
785 def main(args: argparse.Namespace):
@@ -874,7 +879,11 @@ def main(args: argparse.Namespace):
879 else:
880 raise ValueError(f"Unknown dataset: {args.dataset_name}")
881
877 - gootput_config_dict = check_goodput_args(args)
882 + goodput_config_dict = check_goodput_args(args)
883 +
884 + # Avoid GC processing "static" data - reduce pause times.
885 + gc.collect()
886 + gc.freeze()
887
888 benchmark_result = asyncio.run(
889 benchmark(
@@ -896,7 +905,7 @@ def main(args: argparse.Namespace):
905 float(p) for p in args.metric_percentiles.split(",")
906 ],
907 ignore_eos=args.ignore_eos,
899 - gootput_config_dict=gootput_config_dict,
908 + goodput_config_dict=goodput_config_dict,
909 max_concurrency=args.max_concurrency,
910 ))
911