A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Benchmark] More accurate TPOT calc in `benchmark_serving.py` (#12288)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Jan 21, 2025 at 21:46 UTC
222a9dc350d99e111ba09b29576113c9278e6a3e
2 files changed
+66
-46
benchmarks/backend_request_func.py
+27
-16
@@ -35,6 +35,7 @@ class RequestFuncOutput:
35
generated_text: str = ""
36
success: bool = False
37
latency: float = 0.0
38
+ output_tokens: int = 0
39
ttft: float = 0.0 # Time to first token
40
itl: List[float] = field(
41
default_factory=list) # List of inter-token latencies
@@ -156,7 +157,7 @@ async def async_request_trt_llm(
157
timestamp = time.perf_counter()
158
# First token
159
if ttft == 0.0:
159
- ttft = time.perf_counter() - st
160
+ ttft = timestamp - st
161
output.ttft = ttft
162
163
# Decoding phase
@@ -245,6 +246,9 @@ async def async_request_openai_completions(
246
"logprobs": request_func_input.logprobs,
247
"stream": True,
248
"ignore_eos": request_func_input.ignore_eos,
249
+ "stream_options": {
250
+ "include_usage": True,
251
+ },
252
}
253
if request_func_input.extra_body:
254
payload.update(request_func_input.extra_body)
@@ -256,7 +260,6 @@ async def async_request_openai_completions(
260
output.prompt_len = request_func_input.prompt_len
261
262
generated_text = ""
259
- ttft = 0.0
263
st = time.perf_counter()
264
most_recent_timestamp = st
265
try:
@@ -271,15 +274,16 @@ async def async_request_openai_completions(
274
275
chunk = chunk_bytes.decode("utf-8").removeprefix(
276
"data: ")
274
- if chunk == "[DONE]":
275
- latency = time.perf_counter() - st
276
- else:
277
+ if chunk != "[DONE]":
278
data = json.loads(chunk)
279
280
# NOTE: Some completion API might have a last
281
# usage summary response without a token so we
282
# want to check a token was generated
282
- if data["choices"][0]["text"]:
283
+ if choices := data.get("choices"):
284
+ # Note that text could be empty here
285
+ # e.g. for special tokens
286
+ text = choices[0].get("text")
287
timestamp = time.perf_counter()
288
# First token
289
if not first_chunk_received:
@@ -293,7 +297,10 @@ async def async_request_openai_completions(
297
most_recent_timestamp)
298
299
most_recent_timestamp = timestamp
296
- generated_text += data["choices"][0]["text"]
300
+ generated_text += text
301
+ elif usage := data.get("usage"):
302
+ output.output_tokens = usage.get(
303
+ "completion_tokens")
304
if first_chunk_received:
305
output.success = True
306
else:
@@ -302,7 +309,7 @@ async def async_request_openai_completions(
309
"Never received a valid chunk to calculate TTFT."
310
"This response will be marked as failed!")
311
output.generated_text = generated_text
305
- output.latency = latency
312
+ output.latency = most_recent_timestamp - st
313
else:
314
output.error = response.reason or ""
315
output.success = False
@@ -342,6 +349,9 @@ async def async_request_openai_chat_completions(
349
"max_completion_tokens": request_func_input.output_len,
350
"stream": True,
351
"ignore_eos": request_func_input.ignore_eos,
352
+ "stream_options": {
353
+ "include_usage": True,
354
+ },
355
}
356
if request_func_input.extra_body:
357
payload.update(request_func_input.extra_body)
@@ -368,17 +378,15 @@ async def async_request_openai_chat_completions(
378
379
chunk = chunk_bytes.decode("utf-8").removeprefix(
380
"data: ")
371
- if chunk == "[DONE]":
372
- latency = time.perf_counter() - st
373
- else:
381
+ if chunk != "[DONE]":
382
timestamp = time.perf_counter()
383
data = json.loads(chunk)
384
377
- delta = data["choices"][0]["delta"]
378
- if delta.get("content", None):
385
+ if choices := data.get("choices"):
386
+ content = choices[0]["delta"].get("content")
387
# First token
388
if ttft == 0.0:
381
- ttft = time.perf_counter() - st
389
+ ttft = timestamp - st
390
output.ttft = ttft
391
392
# Decoding phase
@@ -386,13 +394,16 @@ async def async_request_openai_chat_completions(
394
output.itl.append(timestamp -
395
most_recent_timestamp)
396
389
- generated_text += delta["content"]
397
+ generated_text += content
398
+ elif usage := data.get("usage"):
399
+ output.output_tokens = usage.get(
400
+ "completion_tokens")
401
402
most_recent_timestamp = timestamp
403
404
output.generated_text = generated_text
405
output.success = True
395
- output.latency = latency
406
+ output.latency = most_recent_timestamp - st
407
else:
408
output.error = response.reason or ""
409
output.success = False
benchmarks/benchmark_serving.py
+39
-30
@@ -25,6 +25,7 @@ On the client side, run:
25
import argparse
26
import asyncio
27
import base64
28
+import gc
29
import io
30
import json
31
import os
@@ -423,7 +424,7 @@ def calculate_metrics(
424
tokenizer: PreTrainedTokenizerBase,
425
selected_percentile_metrics: List[str],
426
selected_percentiles: List[float],
426
- gootput_config_dict: Dict[str, float],
427
+ goodput_config_dict: Dict[str, float],
428
) -> Tuple[BenchmarkMetrics, List[int]]:
429
actual_output_lens: List[int] = []
430
total_input = 0
@@ -436,19 +437,23 @@ def calculate_metrics(
437
e2els: List[float] = []
438
for i in range(len(outputs)):
439
if outputs[i].success:
439
- # We use the tokenizer to count the number of output tokens for all
440
- # serving backends instead of looking at len(outputs[i].itl) since
441
- # multiple output tokens may be bundled together
442
- # Note : this may inflate the output token count slightly
443
- output_len = len(
444
- tokenizer(outputs[i].generated_text,
445
- add_special_tokens=False).input_ids)
440
+ output_len = outputs[i].output_tokens
441
+
442
+ if output_len is None:
443
+ # We use the tokenizer to count the number of output tokens
444
+ # for some serving backends instead of looking at
445
+ # len(outputs[i].itl) since multiple output tokens may be
446
+ # bundled together
447
+ # Note : this may inflate the output token count slightly
448
+ output_len = len(
449
+ tokenizer(outputs[i].generated_text,
450
+ add_special_tokens=False).input_ids)
451
actual_output_lens.append(output_len)
452
total_input += input_requests[i][1]
453
tpot = 0
454
if output_len > 1:
450
- tpot = (outputs[i].latency - outputs[i].ttft) / (output_len -
451
- 1)
455
+ latency_minus_ttft = outputs[i].latency - outputs[i].ttft
456
+ tpot = latency_minus_ttft / (output_len - 1)
457
tpots.append(tpot)
458
# Note: if output_len <= 1, we regard tpot as 0 for goodput
459
all_tpots.append(tpot)
@@ -459,21 +464,21 @@ def calculate_metrics(
464
else:
465
actual_output_lens.append(0)
466
462
- if gootput_config_dict:
467
+ if goodput_config_dict:
468
valid_metrics = []
469
slo_values = []
470
466
- if "ttft" in gootput_config_dict:
471
+ if "ttft" in goodput_config_dict:
472
valid_metrics.append(ttfts)
468
- slo_values.append(gootput_config_dict["ttft"] /
473
+ slo_values.append(goodput_config_dict["ttft"] /
474
MILLISECONDS_TO_SECONDS_CONVERSION)
470
- if "tpot" in gootput_config_dict:
475
+ if "tpot" in goodput_config_dict:
476
valid_metrics.append(all_tpots)
472
- slo_values.append(gootput_config_dict["tpot"] /
477
+ slo_values.append(goodput_config_dict["tpot"] /
478
MILLISECONDS_TO_SECONDS_CONVERSION)
474
- if "e2el" in gootput_config_dict:
479
+ if "e2el" in goodput_config_dict:
480
valid_metrics.append(e2els)
476
- slo_values.append(gootput_config_dict["e2el"] /
481
+ slo_values.append(goodput_config_dict["e2el"] /
482
MILLISECONDS_TO_SECONDS_CONVERSION)
483
484
for req_metric in zip(*valid_metrics):
@@ -537,7 +542,7 @@ async def benchmark(
542
selected_percentile_metrics: List[str],
543
selected_percentiles: List[str],
544
ignore_eos: bool,
540
- gootput_config_dict: Dict[str, float],
545
+ goodput_config_dict: Dict[str, float],
546
max_concurrency: Optional[int],
547
):
548
if backend in ASYNC_REQUEST_FUNCS:
@@ -661,7 +666,7 @@ async def benchmark(
666
tokenizer=tokenizer,
667
selected_percentile_metrics=selected_percentile_metrics,
668
selected_percentiles=selected_percentiles,
664
- gootput_config_dict=gootput_config_dict,
669
+ goodput_config_dict=goodput_config_dict,
670
)
671
672
print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
@@ -673,7 +678,7 @@ async def benchmark(
678
metrics.total_output))
679
print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
680
metrics.request_throughput))
676
- if gootput_config_dict:
681
+ if goodput_config_dict:
682
print("{:<40} {:<10.2f}".format("Request goodput (req/s):",
683
metrics.request_goodput))
684
print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
@@ -688,7 +693,7 @@ async def benchmark(
693
"total_output_tokens": metrics.total_output,
694
"request_throughput": metrics.request_throughput,
695
"request_goodput:":
691
- metrics.request_goodput if gootput_config_dict else None,
696
+ metrics.request_goodput if goodput_config_dict else None,
697
"output_throughput": metrics.output_throughput,
698
"total_token_throughput": metrics.total_token_throughput,
699
"input_lens": [output.prompt_len for output in outputs],
@@ -744,11 +749,11 @@ async def benchmark(
749
750
def check_goodput_args(args):
751
# Check and parse goodput arguments
747
- gootput_config_dict = {}
752
+ goodput_config_dict = {}
753
VALID_NAMES = ["ttft", "tpot", "e2el"]
754
if args.goodput:
750
- gootput_config_dict = parse_goodput(args.goodput)
751
- for slo_name, slo_val in gootput_config_dict.items():
755
+ goodput_config_dict = parse_goodput(args.goodput)
756
+ for slo_name, slo_val in goodput_config_dict.items():
757
if slo_name not in VALID_NAMES:
758
raise ValueError(
759
f"Invalid metric name found, {slo_name}: {slo_val}. "
@@ -759,22 +764,22 @@ def check_goodput_args(args):
764
f"Invalid value found, {slo_name}: {slo_val}. "
765
"The service level objective value should be "
766
"non-negative.")
762
- return gootput_config_dict
767
+ return goodput_config_dict
768
769
770
def parse_goodput(slo_pairs):
766
- gootput_config_dict = {}
771
+ goodput_config_dict = {}
772
try:
773
for slo_pair in slo_pairs:
774
slo_name, slo_val = slo_pair.split(":")
770
- gootput_config_dict[slo_name] = float(slo_val)
775
+ goodput_config_dict[slo_name] = float(slo_val)
776
except ValueError as err:
777
raise argparse.ArgumentTypeError(
778
"Invalid format found for service level objectives. "
779
"Specify service level objectives for goodput as \"KEY:VALUE\" "
780
"pairs, where the key is a metric name, and the value is a "
781
"number in milliseconds.") from err
777
- return gootput_config_dict
782
+ return goodput_config_dict
783
784
785
def main(args: argparse.Namespace):
@@ -874,7 +879,11 @@ def main(args: argparse.Namespace):
879
else:
880
raise ValueError(f"Unknown dataset: {args.dataset_name}")
881
877
- gootput_config_dict = check_goodput_args(args)
882
+ goodput_config_dict = check_goodput_args(args)
883
+
884
+ # Avoid GC processing "static" data - reduce pause times.
885
+ gc.collect()
886
+ gc.freeze()
887
888
benchmark_result = asyncio.run(
889
benchmark(
@@ -896,7 +905,7 @@ def main(args: argparse.Namespace):
905
float(p) for p in args.metric_percentiles.split(",")
906
],
907
ignore_eos=args.ignore_eos,
899
- gootput_config_dict=gootput_config_dict,
908
+ goodput_config_dict=goodput_config_dict,
909
max_concurrency=args.max_concurrency,
910
))
911