@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a904ea78ea

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[benchmark] add peak throughput metrics and plot (#23867)

Signed-off-by: simon-mo <simon.mo@hey.com>

Simon Mo committed Sep 17, 2025 at 22:30 UTC a904ea78eaf7fc3f9b136a1ba6f6f66fb5658496

2 files changed +134 -69

vllm/benchmarks/lib/endpoint_request_func.py

@@ -89,6 +89,7 @@ class RequestFuncOutput:

89 tpot: float = 0.0 # avg next-token latencies

90 prompt_len: int = 0

91 error: str = ""

92 + start_time: float = 0.0

95 async def async_request_openai_completions(

@@ -140,6 +141,7 @@ async def async_request_openai_completions(

141

142 generated_text = ""

143 st = time.perf_counter()

144 + output.start_time = st

145 most_recent_timestamp = st

146 try:

147 async with session.post(url=api_url, json=payload,

@@ -272,6 +274,7 @@ async def async_request_openai_chat_completions(

274 generated_text = ""

275 ttft = 0.0

276 st = time.perf_counter()

277 + output.start_time = st

278 most_recent_timestamp = st

279 try:

280 async with session.post(url=api_url, json=payload,

@@ -396,6 +399,7 @@ async def async_request_openai_audio(

399 generated_text = ""

400 ttft = 0.0

401 st = time.perf_counter()

402 + output.start_time = st

403 most_recent_timestamp = st

404 try:

405 async with session.post(url=api_url,

@@ -475,6 +479,7 @@ async def async_request_openai_embeddings(

479

480 output = RequestFuncOutput()

481 st = time.perf_counter()

482 + output.start_time = st

483 try:

484 async with session.post(

485 url=api_url,

vllm/benchmarks/serve.py

+129 -69

@@ -18,9 +18,11 @@ On the client side, run:

18 import argparse

19 import asyncio

20 import gc

21 +import importlib.util

22 import json

23 import os

24 import random

25 +import shutil

26 import time

27 import warnings

28 from collections.abc import AsyncGenerator, Iterable

@@ -46,6 +48,9 @@ from vllm.transformers_utils.tokenizer import get_tokenizer

49 MILLISECONDS_TO_SECONDS_CONVERSION = 1000

51 +TERM_PLOTLIB_AVAILABLE = ((importlib.util.find_spec("termplotlib") is not None)

52 + and (shutil.which("gnuplot") is not None))

53 +

55 class TaskType(Enum):

56 GENERATION = "generation"

@@ -80,18 +85,23 @@ class BenchmarkMetrics:

85 median_e2el_ms: float

86 std_e2el_ms: float

87 percentiles_e2el_ms: list[tuple[float, float]]

88 + # Max output tokens per second and concurrent requests at that peak

89 + max_output_tokens_per_s: float

90 + max_concurrent_requests: int

91 +

93 @dataclass

94 class EmbedBenchmarkMetrics:

95 completed: int

96 total_input: int

97 request_throughput: float

89 - total_token_throughput :float

98 + total_token_throughput: float

99 mean_e2el_ms: float

100 std_e2el_ms: float

101 median_e2el_ms: float

102 percentiles_e2el_ms: float

103

104 +

105 def _get_current_request_rate(

106 ramp_up_strategy: Optional[Literal["linear", "exponential"]],

107 ramp_up_start_rps: Optional[int],

@@ -150,8 +160,8 @@ async def get_request(

160 assert burstiness > 0, (

161 f"A positive burstiness factor is expected, but given {burstiness}.")

162 # Convert to list to get length for ramp-up calculations

153 - if isinstance(input_requests, Iterable) and not isinstance(

154 - input_requests, list):

163 + if isinstance(input_requests,

164 + Iterable) and not isinstance(input_requests, list):

165 input_requests = list(input_requests)

166

167 total_requests = len(input_requests)

@@ -161,12 +171,9 @@ async def get_request(

171 request_rates = []

172 delay_ts = []

173 for request_index, request in enumerate(input_requests):

164 - current_request_rate = _get_current_request_rate(ramp_up_strategy,

165 - ramp_up_start_rps,

166 - ramp_up_end_rps,

167 - request_index,

168 - total_requests,

169 - request_rate)

174 + current_request_rate = _get_current_request_rate(

175 + ramp_up_strategy, ramp_up_start_rps, ramp_up_end_rps,

176 + request_index, total_requests, request_rate)

177 request_rates.append(current_request_rate)

178 if current_request_rate == float("inf"):

179 delay_ts.append(0)

@@ -206,10 +213,8 @@ async def get_request(

213

214

215 def calculate_metrics_for_embeddings(

209 - outputs: list[RequestFuncOutput],

210 - dur_s: float,

211 - selected_percentiles: list[float]

212 -) -> EmbedBenchmarkMetrics:

216 + outputs: list[RequestFuncOutput], dur_s: float,

217 + selected_percentiles: list[float]) -> EmbedBenchmarkMetrics:

218 """Calculate the metrics for the embedding requests.

219

220 Args:

@@ -242,10 +247,8 @@ def calculate_metrics_for_embeddings(

247 mean_e2el_ms=np.mean(e2els or 0) * 1000,

248 std_e2el_ms=np.std(e2els or 0) * 1000,

249 median_e2el_ms=np.median(e2els or 0) * 1000,

245 - percentiles_e2el_ms=[

246 - (p, np.percentile(e2els or 0, p) * 1000)

247 - for p in selected_percentiles

248 - ],

250 + percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000)

251 + for p in selected_percentiles],

252 )

253 return metrics

254

@@ -336,6 +339,67 @@ def calculate_metrics(

339 "All requests failed. This is likely due to a misconfiguration "

340 "on the benchmark arguments.",

341 stacklevel=2)

342 +

343 + # Calculate max output tokens per second metric

344 + max_output_tokens_per_s = 0.0

345 + max_concurrent_requests = 0

346 +

347 + # Find the time range across all successful requests

348 + successful_outputs = [output for output in outputs if output.success]

349 + if successful_outputs:

350 + min_start_time = min(output.start_time

351 + for output in successful_outputs)

352 + max_end_time = max(output.start_time + output.latency

353 + for output in successful_outputs)

354 +

355 + # Create second buckets (ceiling to ensure we capture all time)

356 + duration_seconds = int(np.ceil(max_end_time - min_start_time)) + 1

357 + tokens_per_second = np.zeros(duration_seconds)

358 + concurrent_requests_per_second = np.zeros(duration_seconds)

359 +

360 + for i, output in enumerate(successful_outputs):

361 + # Calculate token generation timestamp using

362 + # start_time, ttft, and itl

363 + token_times = [output.start_time + output.ttft]

364 + current_time = token_times[0]

365 + for itl_value in output.itl:

366 + current_time += itl_value

367 + token_times.append(current_time)

368 +

369 + # Add tokens to second buckets

370 + for token_time in token_times:

371 + second_bucket = int(token_time - min_start_time)

372 + if 0 <= second_bucket < duration_seconds:

373 + tokens_per_second[second_bucket] += 1

374 +

375 + # Track concurrent requests for each second this request was active

376 + request_start_second = int(output.start_time - min_start_time)

377 + request_end_second = int((output.start_time + output.latency) -

378 + min_start_time)

379 +

380 + for second in range(request_start_second, request_end_second + 1):

381 + concurrent_requests_per_second[second] += 1

382 +

383 + # Find the maximum tokens per second and corresponding

384 + # concurrent requests

385 + if len(tokens_per_second) > 0:

386 + max_output_tokens_per_s = float(np.max(tokens_per_second))

387 + max_concurrent_requests = int(

388 + np.max(concurrent_requests_per_second))

389 +

390 + if TERM_PLOTLIB_AVAILABLE:

391 + import termplotlib as tpl

392 + fig = tpl.figure()

393 + fig.plot(np.arange(len(tokens_per_second)),

394 + tokens_per_second,

395 + title="Output tokens per second")

396 + fig.plot(np.arange(len(concurrent_requests_per_second)),

397 + concurrent_requests_per_second,

398 + title="Concurrent requests per second")

399 + fig.show()

400 + else:

401 + print("tip: install termplotlib and gnuplot to plot the metrics")

402 +

403 metrics = BenchmarkMetrics(

404 completed=completed,

405 total_input=total_input,

@@ -365,6 +429,8 @@ def calculate_metrics(

429 median_e2el_ms=np.median(e2els or 0) * 1000,

430 percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000)

431 for p in selected_percentiles],

432 + max_output_tokens_per_s=max_output_tokens_per_s,

433 + max_concurrent_requests=max_concurrent_requests,

434 )

435

436 return metrics, actual_output_lens

@@ -396,11 +462,8 @@ async def benchmark(

462 ramp_up_end_rps: Optional[int] = None,

463 ready_check_timeout_sec: int = 600,

464 ):

399 - task_type = (

400 - TaskType.EMBEDDING

401 - if api_url.endswith("/v1/embeddings")

402 - else TaskType.GENERATION

403 - )

465 + task_type = (TaskType.EMBEDDING if api_url.endswith("/v1/embeddings") else

466 + TaskType.GENERATION)

467 if endpoint_type in ASYNC_REQUEST_FUNCS:

468 if task_type == TaskType.EMBEDDING:

469 request_func = ASYNC_REQUEST_FUNCS["openai-embeddings"]

@@ -435,14 +498,10 @@ async def benchmark(

498 input_requests[0].multi_modal_data,

499 )

500

438 - assert (

439 - test_mm_content is None

440 - or isinstance(test_mm_content, dict)

441 - or (

442 - isinstance(test_mm_content, list)

443 - and all(isinstance(item, dict) for item in test_mm_content)

444 - )

445 - ), "multi_modal_data must be a dict or list[dict]"

501 + assert (test_mm_content is None or isinstance(test_mm_content, dict)

502 + or (isinstance(test_mm_content, list)

503 + and all(isinstance(item, dict) for item in test_mm_content))

504 + ), "multi_modal_data must be a dict or list[dict]"

505 test_input = RequestFuncInput(

506 model=model_id,

507 model_name=model_name,

@@ -488,13 +547,13 @@ async def benchmark(

547 ignore_eos=ignore_eos,

548 extra_headers=extra_headers,

549 extra_body=extra_body)

491 - profile_output = await request_func(

492 - request_func_input=profile_input, session=session)

550 + profile_output = await request_func(request_func_input=profile_input,

551 + session=session)

552 if profile_output.success:

553 print("Profiler started")

554

496 - distribution = ("Poisson process" if burstiness == 1.0

497 - else "Gamma distribution")

555 + distribution = ("Poisson process"

556 + if burstiness == 1.0 else "Gamma distribution")

557

558 if ramp_up_strategy is not None:

559 print(f"Traffic ramp-up strategy: {ramp_up_strategy}.")

@@ -562,18 +621,20 @@ async def benchmark(

621 req_lora_module = next(lora_modules)

622 req_model_id, req_model_name = req_lora_module, req_lora_module

623

565 - request_func_input = RequestFuncInput(model=req_model_id,

566 - model_name=req_model_name,

567 - prompt=prompt,

568 - api_url=api_url,

569 - prompt_len=prompt_len,

570 - output_len=output_len,

571 - logprobs=logprobs,

572 - multi_modal_content=mm_content,

573 - ignore_eos=ignore_eos,

574 - extra_headers=extra_headers,

575 - extra_body=extra_body,

576 - request_id=request_id,)

624 + request_func_input = RequestFuncInput(

625 + model=req_model_id,

626 + model_name=req_model_name,

627 + prompt=prompt,

628 + api_url=api_url,

629 + prompt_len=prompt_len,

630 + output_len=output_len,

631 + logprobs=logprobs,

632 + multi_modal_content=mm_content,

633 + ignore_eos=ignore_eos,

634 + extra_headers=extra_headers,

635 + extra_body=extra_body,

636 + request_id=request_id,

637 + )

638 tasks.append(

639 asyncio.create_task(

640 limited_request_func(request_func_input=request_func_input,

@@ -615,19 +676,21 @@ async def benchmark(

676 benchmark_duration))

677 print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))

678 if isinstance(metrics, BenchmarkMetrics):

618 - print("{:<40} {:<10}".format(

619 - "Total generated tokens:", metrics.total_output))

679 + print("{:<40} {:<10}".format("Total generated tokens:",

680 + metrics.total_output))

681 print("{:<40} {:<10.2f}".format("Request throughput (req/s):",

682 metrics.request_throughput))

683 if goodput_config_dict:

684 print("{:<40} {:<10.2f}".format("Request goodput (req/s):",

685 metrics.request_goodput))

686 if isinstance(metrics, BenchmarkMetrics):

626 - print(

627 - "{:<40} {:<10.2f}".format(

628 - "Output token throughput (tok/s):", metrics.output_throughput

629 - )

630 - )

687 + print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",

688 + metrics.output_throughput))

689 + print("{:<40} {:<10.2f}".format(

690 + "Peak output token throughput (tok/s):",

691 + metrics.max_output_tokens_per_s))

692 + print("{:<40} {:<10.2f}".format("Peak concurrent requests:",

693 + metrics.max_concurrent_requests))

694 print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):",

695 metrics.total_token_throughput))

696

@@ -648,6 +711,8 @@ async def benchmark(

711 "itls": [output.itl for output in outputs],

712 "generated_texts": [output.generated_text for output in outputs],

713 "errors": [output.error for output in outputs],

714 + "max_output_tokens_per_s": metrics.max_output_tokens_per_s,

715 + "max_concurrent_requests": metrics.max_concurrent_requests,

716 }

717 else:

718 result = {

@@ -697,8 +762,8 @@ async def benchmark(

762

763 if task_type == TaskType.GENERATION:

764 process_one_metric("ttft", "TTFT", "Time to First Token")

700 - process_one_metric(

701 - "tpot", "TPOT", "Time per Output Token (excl. 1st token)")

765 + process_one_metric("tpot", "TPOT",

766 + "Time per Output Token (excl. 1st token)")

767 process_one_metric("itl", "ITL", "Inter-token Latency")

768 process_one_metric("e2el", "E2EL", "End-to-end Latency")

769

@@ -714,8 +779,8 @@ async def benchmark(

779 output_len=test_output_len,

780 logprobs=logprobs,

781 )

717 - profile_output = await request_func(

718 - request_func_input=profile_input, session=session)

782 + profile_output = await request_func(request_func_input=profile_input,

783 + session=session)

784 if profile_output.success:

785 print("Profiler stopped")

786

@@ -851,7 +916,8 @@ def add_cli_args(parser: argparse.ArgumentParser):

916 parser.add_argument(

917 "--tokenizer",

918 type=str,

854 - help="Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501

919 + help=

920 + "Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501

921 )

922 parser.add_argument("--use-beam-search", action="store_true")

923 parser.add_argument(

@@ -982,7 +1048,6 @@ def add_cli_args(parser: argparse.ArgumentParser):

1048 help="Specify the prefix of request id.",

1049 )

1050

985 -

1051 sampling_group = parser.add_argument_group("sampling parameters")

1052 sampling_group.add_argument(

1053 "--top-p",

@@ -1047,8 +1112,7 @@ def add_cli_args(parser: argparse.ArgumentParser):

1112 help="The ramp-up strategy. This would be used to "

1113 "ramp up the request rate from initial RPS to final "

1114 "RPS rate (specified by --ramp-up-start-rps and "

1050 - "--ramp-up-end-rps.) over the duration of the benchmark."

1051 - )

1115 + "--ramp-up-end-rps.) over the duration of the benchmark.")

1116 parser.add_argument(

1117 "--ramp-up-start-rps",

1118 type=int,

@@ -1087,13 +1151,11 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:

1151 raise ValueError(

1152 "When using ramp-up, do not specify --request-rate. "

1153 "The request rate will be controlled by ramp-up parameters. "

1090 - "Please remove the --request-rate argument."

1091 - )

1154 + "Please remove the --request-rate argument.")

1155 if args.ramp_up_start_rps is None or args.ramp_up_end_rps is None:

1156 raise ValueError(

1157 "When using --ramp-up-strategy, both --ramp-up-start-rps and "

1095 - "--ramp-up-end-rps must be specified"

1096 - )

1158 + "--ramp-up-end-rps must be specified")

1159 if args.ramp_up_start_rps < 0 or args.ramp_up_end_rps < 0:

1160 raise ValueError("Ramp-up start and end RPS must be non-negative")

1161 if args.ramp_up_start_rps > args.ramp_up_end_rps:

@@ -1127,8 +1189,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:

1189 headers[kvstring[0].strip()] = kvstring[1].strip()

1190 else:

1191 raise ValueError(

1130 - "Invalid header format. Please use KEY=VALUE format."

1131 - )

1192 + "Invalid header format. Please use KEY=VALUE format.")

1193

1194 tokenizer = get_tokenizer(tokenizer_id,

1195 tokenizer_mode=tokenizer_mode,

@@ -1215,8 +1276,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:

1276 result_json[kvstring[0].strip()] = kvstring[1].strip()

1277 else:

1278 raise ValueError(

1218 - "Invalid metadata format. Please use KEY=VALUE format."

1219 - )

1279 + "Invalid metadata format. Please use KEY=VALUE format.")

1280

1281 # Traffic

1282 result_json["request_rate"] = (args.request_rate if args.request_rate