| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import argparse |
| 5 | import json |
| 6 | import os |
| 7 | import shlex |
| 8 | from importlib import util |
| 9 | from pathlib import Path |
| 10 | from typing import Any |
| 11 | |
| 12 | import pandas as pd |
| 13 | import psutil |
| 14 | import regex as re |
| 15 | from tabulate import tabulate |
| 16 | |
| 17 | # latency results and the keys that will be printed into markdown |
| 18 | latency_results = [] |
| 19 | latency_column_mapping = { |
| 20 | "test_name": "Test name", |
| 21 | "gpu_type": "GPU", |
| 22 | "avg_latency": "Mean latency (ms)", |
| 23 | # "P10": "P10 (s)", |
| 24 | # "P25": "P25 (s)", |
| 25 | "P50": "Median latency (ms)", |
| 26 | # "P75": "P75 (s)", |
| 27 | # "P90": "P90 (s)", |
| 28 | "P99": "P99 latency (ms)", |
| 29 | } |
| 30 | |
| 31 | # throughput tests and the keys that will be printed into markdown |
| 32 | throughput_results = [] |
| 33 | throughput_results_column_mapping = { |
| 34 | "test_name": "Test name", |
| 35 | "gpu_type": "GPU", |
| 36 | "num_requests": "# of req.", |
| 37 | "total_num_tokens": "Total # of tokens", |
| 38 | "elapsed_time": "Elapsed time (s)", |
| 39 | "requests_per_second": "Tput (req/s)", |
| 40 | "tokens_per_second": "Tput (tok/s)", |
| 41 | } |
| 42 | |
| 43 | # serving results and the keys that will be printed into markdown |
| 44 | serving_results = [] |
| 45 | serving_column_mapping = { |
| 46 | "test_name": "Test name", |
| 47 | "model_id": "Model", |
| 48 | "dataset_name": "Dataset Name", |
| 49 | "input_len": "Input Len", |
| 50 | "output_len": "Output Len", |
| 51 | "tp_size": "TP Size", |
| 52 | "pp_size": "PP Size", |
| 53 | "dtype": "dtype", |
| 54 | "gpu_type": "GPU", |
| 55 | "completed": "# of req.", |
| 56 | "qps": "qps", |
| 57 | "max_concurrency": "# of max concurrency.", |
| 58 | "request_throughput": "Tput (req/s)", |
| 59 | "total_token_throughput": "Total Token Tput (tok/s)", |
| 60 | "output_throughput": "Output Tput (tok/s)", |
| 61 | # "total_input_tokens": "Total input tokens", |
| 62 | # "total_output_tokens": "Total output tokens", |
| 63 | "mean_ttft_ms": "Mean TTFT (ms)", |
| 64 | "median_ttft_ms": "Median TTFT (ms)", |
| 65 | "p99_ttft_ms": "P99 TTFT (ms)", |
| 66 | "std_ttft_ms": "STD TTFT (ms)", |
| 67 | "mean_tpot_ms": "Mean TPOT (ms)", |
| 68 | "median_tpot_ms": "Median", |
| 69 | "p99_tpot_ms": "P99", |
| 70 | "std_tpot_ms": "STD TPOT (ms)", |
| 71 | "mean_itl_ms": "Mean ITL (ms)", |
| 72 | "median_itl_ms": "Median ITL (ms)", |
| 73 | "p99_itl_ms": "P99 ITL (ms)", |
| 74 | } |
| 75 | |
| 76 | |
| 77 | def read_markdown(file): |
| 78 | if os.path.exists(file): |
| 79 | with open(file) as f: |
| 80 | return f.read() + "\n" |
| 81 | else: |
| 82 | return f"{file} not found.\n" |
| 83 | |
| 84 | |
| 85 | def results_to_json(latency, throughput, serving): |
| 86 | return json.dumps( |
| 87 | { |
| 88 | "latency": latency.to_dict(), |
| 89 | "throughput": throughput.to_dict(), |
| 90 | "serving": serving.to_dict(), |
| 91 | } |
| 92 | ) |
| 93 | |
| 94 | |
| 95 | def get_size_with_unit(bytes, suffix="B"): |
| 96 | """Scale bytes to its proper format |
| 97 | e.g: |
| 98 | 1253656 => '1.20MB' |
| 99 | 1253656678 => '1.17GB' |
| 100 | """ |
| 101 | factor = 1024 |
| 102 | for unit in ["", "K", "M", "G", "T", "P"]: |
| 103 | if bytes < factor: |
| 104 | return f"{bytes:.2f}{unit}{suffix}" |
| 105 | bytes /= factor |
| 106 | |
| 107 | |
| 108 | def _coerce(val: str) -> Any: |
| 109 | """Best-effort type coercion from string to Python types.""" |
| 110 | low = val.lower() |
| 111 | if low == "null": |
| 112 | return None |
| 113 | if low == "true": |
| 114 | return True |
| 115 | if low == "false": |
| 116 | return False |
| 117 | # integers |
| 118 | if re.fullmatch(r"[+-]?\d+", val): |
| 119 | try: |
| 120 | return int(val) |
| 121 | except ValueError: |
| 122 | pass |
| 123 | # floats (keep 'inf'/'-inf'/'nan' as strings) |
| 124 | if re.fullmatch(r"[+-]?\d*\.\d+", val): |
| 125 | try: |
| 126 | return float(val) |
| 127 | except ValueError: |
| 128 | pass |
| 129 | return val |
| 130 | |
| 131 | |
| 132 | def parse_client_command(cmd: str) -> dict[str, Any]: |
| 133 | """Parse the client_command shell string into {executable, script, args}.""" |
| 134 | toks = shlex.split(cmd) |
| 135 | if len(toks) < 2: |
| 136 | raise ValueError("client_command must include an executable and a script") |
| 137 | executable, script = toks[0], toks[1] |
| 138 | args: dict[str, Any] = {} |
| 139 | |
| 140 | i = 2 |
| 141 | while i < len(toks): |
| 142 | t = toks[i] |
| 143 | if t.startswith("--"): |
| 144 | # --key=value or --key (value) or boolean flag |
| 145 | if "=" in t: |
| 146 | key, val = t.split("=", 1) |
| 147 | if key == "--metadata": |
| 148 | md = {} |
| 149 | if val: |
| 150 | if "=" in val: |
| 151 | k, v = val.split("=", 1) |
| 152 | md[k] = _coerce(v) |
| 153 | else: |
| 154 | md[val] = True |
| 155 | args[key] = md |
| 156 | else: |
| 157 | args[key] = _coerce(val) |
| 158 | i += 1 |
| 159 | continue |
| 160 | |
| 161 | key = t |
| 162 | |
| 163 | # Special: consume metadata k=v pairs until next --flag |
| 164 | if key == "--metadata": |
| 165 | i += 1 |
| 166 | md = {} |
| 167 | while i < len(toks) and not toks[i].startswith("--"): |
| 168 | pair = toks[i] |
| 169 | if "=" in pair: |
| 170 | k, v = pair.split("=", 1) |
| 171 | md[k] = _coerce(v) |
| 172 | else: |
| 173 | md[pair] = True |
| 174 | i += 1 |
| 175 | args[key] = md |
| 176 | continue |
| 177 | |
| 178 | # Standard: check if next token is a value (not a flag) |
| 179 | if i + 1 < len(toks) and not toks[i + 1].startswith("--"): |
| 180 | args[key] = _coerce(toks[i + 1]) |
| 181 | i += 2 |
| 182 | else: |
| 183 | # lone flag -> True |
| 184 | args[key] = True |
| 185 | i += 1 |
| 186 | else: |
| 187 | # unexpected positional; skip |
| 188 | i += 1 |
| 189 | |
| 190 | return {"executable": executable, "script": script, "args": args} |
| 191 | |
| 192 | |
| 193 | if __name__ == "__main__": |
| 194 | parser = argparse.ArgumentParser() |
| 195 | parser.add_argument( |
| 196 | "-r", |
| 197 | "--result", |
| 198 | type=str, |
| 199 | default="results", |
| 200 | help="Folder name for benchmark output results.", |
| 201 | ) |
| 202 | args = parser.parse_args() |
| 203 | results_folder = Path(args.result) |
| 204 | if not results_folder.exists(): |
| 205 | raise FileNotFoundError(f"results folder does not exist: {results_folder}") |
| 206 | # collect results |
| 207 | for test_file in results_folder.glob("*.json"): |
| 208 | with open(test_file) as f: |
| 209 | raw_result = json.loads(f.read()) |
| 210 | |
| 211 | if "serving" in str(test_file): |
| 212 | # this result is generated via `vllm bench serve` command |
| 213 | # attach the benchmarking command to raw_result |
| 214 | try: |
| 215 | with open(test_file.with_suffix(".commands")) as f: |
| 216 | command = json.loads(f.read()) |
| 217 | except OSError as e: |
| 218 | print(e) |
| 219 | continue |
| 220 | # Parse Server Command Arg |
| 221 | out: dict[str, Any] = { |
| 222 | "server_command": parse_client_command(command["server_command"]) |
| 223 | } |
| 224 | parse_args = [ |
| 225 | "--tensor-parallel-size", |
| 226 | "--pipeline-parallel-size", |
| 227 | "--dtype", |
| 228 | ] |
| 229 | col_mapping = ["tp_size", "pp_size", "dtype"] |
| 230 | for index, arg in enumerate(parse_args): |
| 231 | if arg in out["server_command"]["args"]: |
| 232 | raw_result.update( |
| 233 | {col_mapping[index]: out["server_command"]["args"][arg]} |
| 234 | ) |
| 235 | |
| 236 | # Parse Client Command Arg |
| 237 | out: dict[str, Any] = { |
| 238 | "client_command": parse_client_command(command["client_command"]) |
| 239 | } |
| 240 | parse_args = [ |
| 241 | "--dataset-name", |
| 242 | "--random-input-len", |
| 243 | "--random-output-len", |
| 244 | "--request-rate", |
| 245 | ] |
| 246 | col_mapping = ["dataset_name", "input_len", "output_len", "qps"] |
| 247 | |
| 248 | for index, arg in enumerate(parse_args): |
| 249 | if arg in out["client_command"]["args"]: |
| 250 | raw_result.update( |
| 251 | {col_mapping[index]: out["client_command"]["args"][arg]} |
| 252 | ) |
| 253 | # Add Server, Client command |
| 254 | raw_result.update(command) |
| 255 | |
| 256 | # update the test name of this result |
| 257 | raw_result.update({"test_name": test_file.stem}) |
| 258 | # add the result to raw_result |
| 259 | serving_results.append(raw_result) |
| 260 | continue |
| 261 | |
| 262 | elif "latency" in f.name: |
| 263 | # this result is generated via `vllm bench latency` command |
| 264 | |
| 265 | # attach the benchmarking command to raw_result |
| 266 | try: |
| 267 | with open(test_file.with_suffix(".commands")) as f: |
| 268 | command = json.loads(f.read()) |
| 269 | except OSError as e: |
| 270 | print(e) |
| 271 | continue |
| 272 | |
| 273 | raw_result.update(command) |
| 274 | |
| 275 | # update the test name of this result |
| 276 | raw_result.update({"test_name": test_file.stem}) |
| 277 | |
| 278 | # get different percentiles |
| 279 | for perc in [10, 25, 50, 75, 90, 99]: |
| 280 | # Multiply 1000 to convert the time unit from s to ms |
| 281 | raw_result.update( |
| 282 | {f"P{perc}": 1000 * raw_result["percentiles"][str(perc)]} |
| 283 | ) |
| 284 | raw_result["avg_latency"] = raw_result["avg_latency"] * 1000 |
| 285 | |
| 286 | # add the result to raw_result |
| 287 | latency_results.append(raw_result) |
| 288 | continue |
| 289 | |
| 290 | elif "throughput" in f.name: |
| 291 | # this result is generated via `vllm bench throughput` command |
| 292 | |
| 293 | # attach the benchmarking command to raw_result |
| 294 | try: |
| 295 | with open(test_file.with_suffix(".commands")) as f: |
| 296 | command = json.loads(f.read()) |
| 297 | except OSError as e: |
| 298 | print(e) |
| 299 | continue |
| 300 | |
| 301 | raw_result.update(command) |
| 302 | |
| 303 | # update the test name of this result |
| 304 | raw_result.update({"test_name": test_file.stem}) |
| 305 | |
| 306 | # add the result to raw_result |
| 307 | throughput_results.append(raw_result) |
| 308 | continue |
| 309 | |
| 310 | print(f"Skipping {test_file}") |
| 311 | |
| 312 | latency_results = pd.DataFrame.from_dict(latency_results) |
| 313 | serving_results = pd.DataFrame.from_dict(serving_results) |
| 314 | throughput_results = pd.DataFrame.from_dict(throughput_results) |
| 315 | |
| 316 | svmem = psutil.virtual_memory() |
| 317 | platform_data = { |
| 318 | "Physical cores": [psutil.cpu_count(logical=False)], |
| 319 | "Total cores": [psutil.cpu_count(logical=True)], |
| 320 | "Total Memory": [get_size_with_unit(svmem.total)], |
| 321 | } |
| 322 | |
| 323 | if util.find_spec("numa") is not None: |
| 324 | from numa import info |
| 325 | |
| 326 | platform_data["Total NUMA nodes"] = [info.get_num_configured_nodes()] |
| 327 | |
| 328 | if util.find_spec("cpuinfo") is not None: |
| 329 | from cpuinfo import get_cpu_info |
| 330 | |
| 331 | platform_data["CPU Brand"] = [get_cpu_info()["brand_raw"]] |
| 332 | |
| 333 | platform_results = pd.DataFrame.from_dict( |
| 334 | platform_data, orient="index", columns=["Platform Info"] |
| 335 | ) |
| 336 | |
| 337 | raw_results_json = results_to_json( |
| 338 | latency_results, throughput_results, serving_results |
| 339 | ) |
| 340 | |
| 341 | # remapping the key, for visualization purpose |
| 342 | if not latency_results.empty: |
| 343 | latency_results = latency_results[list(latency_column_mapping.keys())].rename( |
| 344 | columns=latency_column_mapping |
| 345 | ) |
| 346 | if not serving_results.empty: |
| 347 | valid_columns = [ |
| 348 | col for col in serving_column_mapping if col in serving_results.columns |
| 349 | ] |
| 350 | serving_results = serving_results[valid_columns].rename( |
| 351 | columns=serving_column_mapping |
| 352 | ) |
| 353 | if not throughput_results.empty: |
| 354 | throughput_results = throughput_results[ |
| 355 | list(throughput_results_column_mapping.keys()) |
| 356 | ].rename(columns=throughput_results_column_mapping) |
| 357 | |
| 358 | processed_results_json = results_to_json( |
| 359 | latency_results, throughput_results, serving_results |
| 360 | ) |
| 361 | |
| 362 | for df in [latency_results, serving_results, throughput_results]: |
| 363 | if df.empty: |
| 364 | continue |
| 365 | |
| 366 | # Sort all dataframes by their respective "Test name" columns |
| 367 | df.sort_values(by="Test name", inplace=True) |
| 368 | |
| 369 | # The GPUs sometimes come in format of "GPUTYPE\nGPUTYPE\n...", |
| 370 | # we want to turn it into "8xGPUTYPE" |
| 371 | df["GPU"] = df["GPU"].apply( |
| 372 | lambda x: "{}x{}".format(len(x.split("\n")), x.split("\n")[0]) |
| 373 | ) |
| 374 | |
| 375 | # get markdown tables |
| 376 | latency_md_table = tabulate( |
| 377 | latency_results, headers="keys", tablefmt="pipe", showindex=False |
| 378 | ) |
| 379 | serving_md_table = tabulate( |
| 380 | serving_results, headers="keys", tablefmt="pipe", showindex=False |
| 381 | ) |
| 382 | throughput_md_table = tabulate( |
| 383 | throughput_results, headers="keys", tablefmt="pipe", showindex=False |
| 384 | ) |
| 385 | platform_md_table = tabulate( |
| 386 | platform_results, headers="keys", tablefmt="pipe", showindex=True |
| 387 | ) |
| 388 | |
| 389 | # document the result |
| 390 | md_file = "benchmark_results.md" |
| 391 | json_file = "benchmark_results.json" |
| 392 | with open(results_folder / md_file, "w") as f: |
| 393 | results = read_markdown( |
| 394 | "../.buildkite/performance-benchmarks/" |
| 395 | "performance-benchmarks-descriptions.md" |
| 396 | ) |
| 397 | results = results.format( |
| 398 | latency_tests_markdown_table=latency_md_table, |
| 399 | throughput_tests_markdown_table=throughput_md_table, |
| 400 | serving_tests_markdown_table=serving_md_table, |
| 401 | platform_markdown_table=platform_md_table, |
| 402 | benchmarking_results_in_json_string=processed_results_json, |
| 403 | ) |
| 404 | f.write(results) |
| 405 | |
| 406 | # document benchmarking results in json |
| 407 | with open(results_folder / json_file, "w") as f: |
| 408 | results = ( |
| 409 | latency_results.to_dict(orient="records") |
| 410 | + throughput_results.to_dict(orient="records") |
| 411 | + serving_results.to_dict(orient="records") |
| 412 | ) |
| 413 | f.write(json.dumps(results)) |