@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 55a1a9563a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 10:00 UTC

Vllm CPU benchmark suite improvement (#34128)

Signed-off-by: louie-tsai <louie.tsai@intel.com>

Louie Tsai committed Feb 12, 2026 at 00:04 UTC 55a1a9563a7f8600cdc336e76d2074cef8ffe8e5
6 files changed +802 -254
.buildkite/performance-benchmarks/scripts/compare-json-results.py
+368 -77
@@ -9,8 +9,10 @@ import json
9 import os
10 from dataclasses import dataclass
11 from importlib import util
12 +from pathlib import Path
13
14 import pandas as pd
15 +import regex as re
16
17 pd.options.display.float_format = "{:.2f}".format
18 plotly_found = util.find_spec("plotly.express") is not None
@@ -275,6 +277,131 @@ def _apply_two_decimals(
277 return styler.format({c: "{:.2f}" for c in num_cols}, na_rep="")
278
279
280 +# -----------------------------
281 +# Export helpers (Excel + CSV)
282 +# -----------------------------
283 +def _sanitize_sheet_name(name: str) -> str:
284 + """
285 + Excel sheet constraints:
286 + - max 31 chars
287 + - cannot contain: : \ / ? * [ ]
288 + - cannot be empty
289 + """
290 + name = "sheet" if name is None else str(name)
291 + name = re.sub(r"[:\\/?*\[\]]", "_", name)
292 + name = name.strip().strip("'")
293 + name = re.sub(r"\s+", " ", name)
294 + if not name:
295 + name = "sheet"
296 + return name[:31]
297 +
298 +
299 +def _group_to_sheet_base(group_cols: list[str], gkey_tuple) -> str:
300 + d = dict(zip(group_cols, gkey_tuple))
301 + model = d.get("Model", "model")
302 + model_short = str(model).split("/")[-1]
303 + ilen = d.get("Input Len", "")
304 + olen = d.get("Output Len", "")
305 + lens = f"_{ilen}x{olen}" if ilen != "" and olen != "" else ""
306 + return _sanitize_sheet_name(f"{model_short}{lens}")
307 +
308 +
309 +def _write_tables_to_excel_sheet(
310 + writer: pd.ExcelWriter, sheet: str, blocks: list[tuple[str, pd.DataFrame]]
311 +):
312 + startrow = 0
313 + for title, df in blocks:
314 + pd.DataFrame([[title]]).to_excel(
315 + writer, sheet_name=sheet, index=False, header=False, startrow=startrow
316 + )
317 + startrow += 1
318 + df.to_excel(writer, sheet_name=sheet, index=False, startrow=startrow)
319 + startrow += len(df) + 3
320 +
321 +
322 +def _safe_filename(s: str) -> str:
323 + s = re.sub(r"[^\w\-.]+", "_", str(s).strip())
324 + return s[:180] if len(s) > 180 else s
325 +
326 +
327 +# -----------------------------
328 +# vLLM environment export helper
329 +# -----------------------------
330 +def _parse_vllm_env_txt(env_path: Path) -> pd.DataFrame:
331 + """Parse vllm_env.txt into a flat table (Section, Key, Value).
332 +
333 + Supports:
334 + - section headers as standalone lines (no ':' or '=')
335 + - key-value lines like 'OS: Ubuntu ...'
336 + - env var lines like 'HF_HOME=/data/hf'
337 + """
338 + lines = env_path.read_text(encoding="utf-8", errors="replace").splitlines()
339 + section = "General"
340 + rows: list[dict] = []
341 +
342 + def set_section(s: str):
343 + nonlocal section
344 + s = (s or "").strip()
345 + if s:
346 + section = s
347 +
348 + for raw in lines:
349 + stripped = raw.strip()
350 + if not stripped:
351 + continue
352 + # divider lines like =====
353 + if set(stripped) <= {"="}:
354 + continue
355 +
356 + # section header heuristic: short standalone line
357 + if ":" not in stripped and "=" not in stripped and len(stripped) <= 64:
358 + if stripped.lower().startswith("collecting environment information"):
359 + continue
360 + set_section(stripped)
361 + continue
362 +
363 + # env var style: KEY=VALUE (and not a URL with :)
364 + if "=" in stripped and ":" not in stripped:
365 + k, v = stripped.split("=", 1)
366 + k = k.strip()
367 + v = v.strip()
368 + if k:
369 + rows.append({"Section": section, "Key": k, "Value": v})
370 + continue
371 +
372 + # key: value
373 + if ":" in stripped:
374 + k, v = stripped.split(":", 1)
375 + k = k.strip()
376 + v = v.strip()
377 + if k:
378 + rows.append({"Section": section, "Key": k, "Value": v})
379 + continue
380 +
381 + return pd.DataFrame(rows, columns=["Section", "Key", "Value"])
382 +
383 +
384 +def _load_env_df_for_inputs(args, files: list[str]) -> pd.DataFrame | None:
385 + """Load vllm_env.txt next to the *original* input JSON file.
386 +
387 + Note: when only one -f is provided, the script may split JSON into ./splits/...,
388 + but vllm_env.txt typically lives next to the original benchmark_results.json.
389 + """
390 + base_dir: Path | None = None
391 + if getattr(args, "file", None):
392 + base_dir = Path(args.file[0]).resolve().parent
393 + elif files:
394 + base_dir = Path(files[0]).resolve().parent
395 + if base_dir is None:
396 + return None
397 +
398 + env_path = base_dir / "vllm_env.txt"
399 + if not env_path.exists():
400 + return None
401 + df = _parse_vllm_env_txt(env_path)
402 + return df
403 +
404 +
405 # -----------------------------
406 # Valid max concurrency summary helpers
407 # -----------------------------
@@ -428,7 +555,6 @@ def build_valid_max_concurrency_summary_html(
555
556 summary_df = pd.DataFrame(rows)
557
431 - # --- Coerce numeric columns so Styler doesn't miss them due to object dtype ---
558 for c in summary_df.columns:
559 if c == "Configuration":
560 continue
@@ -436,12 +562,10 @@ def build_valid_max_concurrency_summary_html(
562
563 both_col = f"Max {conc_col} (Both)"
564
439 - # --- Strict 2-decimal formatting for ALL non-Configuration columns ---
565 formatters = {}
566 for c in summary_df.columns:
567 if c == "Configuration":
568 continue
444 - # default argument binds per-column formatter correctly
569 formatters[c] = lambda v: "" if pd.isna(v) else f"{float(v):.2f}"
570
571 styler = summary_df.style.format(formatters)
@@ -460,6 +584,95 @@ def build_valid_max_concurrency_summary_html(
584 return title + styler.to_html(table_attributes='border="1" class="dataframe"')
585
586
587 +def build_valid_max_concurrency_summary_df(
588 + tput_group_df: pd.DataFrame | None,
589 + ttft_group_df: pd.DataFrame | None,
590 + tpot_group_df: pd.DataFrame | None,
591 + conc_col: str,
592 + args,
593 +) -> pd.DataFrame | None:
594 + if ttft_group_df is None and tpot_group_df is None:
595 + return None
596 +
597 + ttft_cols = (
598 + _config_value_columns(ttft_group_df, conc_col)
599 + if ttft_group_df is not None
600 + else []
601 + )
602 + tpot_cols = (
603 + _config_value_columns(tpot_group_df, conc_col)
604 + if tpot_group_df is not None
605 + else []
606 + )
607 + tput_cols = (
608 + _config_value_columns(tput_group_df, conc_col)
609 + if tput_group_df is not None
610 + else []
611 + )
612 +
613 + if ttft_group_df is not None and tpot_group_df is not None:
614 + cfg_cols = [c for c in ttft_cols if c in tpot_cols]
615 + if tput_group_df is not None:
616 + cfg_cols = [c for c in cfg_cols if c in tput_cols] or cfg_cols
617 + else:
618 + cfg_cols = ttft_cols or tpot_cols
619 +
620 + if not cfg_cols:
621 + cfg_cols = sorted(set(ttft_cols) | set(tpot_cols) | set(tput_cols), key=str)
622 +
623 + rows = []
624 + for cfg in cfg_cols:
625 + ttft_max = (
626 + _max_concurrency_ok(ttft_group_df, conc_col, cfg, args.ttft_max_ms)
627 + if ttft_group_df is not None
628 + else pd.NA
629 + )
630 + tpot_max = (
631 + _max_concurrency_ok(tpot_group_df, conc_col, cfg, args.tpot_max_ms)
632 + if tpot_group_df is not None
633 + else pd.NA
634 + )
635 + both = (
636 + pd.NA
637 + if (pd.isna(ttft_max) or pd.isna(tpot_max))
638 + else min(ttft_max, tpot_max)
639 + )
640 +
641 + tput_at_both = (
642 + _value_at_concurrency(tput_group_df, conc_col, cfg, both)
643 + if tput_group_df is not None
644 + else pd.NA
645 + )
646 + ttft_at_both = (
647 + _value_at_concurrency(ttft_group_df, conc_col, cfg, both)
648 + if ttft_group_df is not None
649 + else pd.NA
650 + )
651 + tpot_at_both = (
652 + _value_at_concurrency(tpot_group_df, conc_col, cfg, both)
653 + if tpot_group_df is not None
654 + else pd.NA
655 + )
656 +
657 + rows.append(
658 + {
659 + "Configuration": cfg,
660 + f"Max {conc_col} (TTFT ≤ {args.ttft_max_ms:g} ms)": ttft_max,
661 + f"Max {conc_col} (TPOT ≤ {args.tpot_max_ms:g} ms)": tpot_max,
662 + f"Max {conc_col} (Both)": both,
663 + "Output Tput @ Both (tok/s)": tput_at_both,
664 + "TTFT @ Both (ms)": ttft_at_both,
665 + "TPOT @ Both (ms)": tpot_at_both,
666 + }
667 + )
668 +
669 + df = pd.DataFrame(rows)
670 + for c in df.columns:
671 + if c != "Configuration":
672 + df[c] = pd.to_numeric(df[c], errors="coerce")
673 + return df
674 +
675 +
676 # -----------------------------
677 # Plot helper
678 # -----------------------------
@@ -537,6 +750,21 @@ def build_parser() -> argparse.ArgumentParser:
750 default=100.0,
751 help="Reference limit for TPOT plots (ms)",
752 )
753 +
754 + # ---- NEW: export options ----
755 + parser.add_argument(
756 + "--excel-out",
757 + type=str,
758 + default="perf_comparison.xlsx",
759 + help="Write one sheet per (Model, Dataset, Input Len, Output Len).",
760 + )
761 + parser.add_argument(
762 + "--csv-out-dir",
763 + type=str,
764 + default="",
765 + help="If set, write per-group per-metric CSVs into this directory.",
766 + )
767 +
768 return parser
769
770
@@ -657,7 +885,6 @@ def maybe_write_plot(
885 markers=True,
886 )
887
660 - # Ensure plot hover + y tick labels are also 2 decimals.
888 fig.update_traces(hovertemplate="%{y:.2f}<extra></extra>")
889 fig.update_yaxes(tickformat=".2f")
890
@@ -730,87 +957,151 @@ def write_report_group_first(
957 for metric_label, (df, _) in metric_cache.items()
958 }
959
733 - with open("perf_comparison.html", "w", encoding="utf-8") as main_fh:
734 - main_fh.write('<meta charset="utf-8">\n')
735 - for gkey in group_keys:
736 - gkey_tuple = normalize_group_key(gkey)
737 - suffix = build_group_suffix(group_cols_canonical, gkey_tuple)
738 - sub_path = group_filename(gkey_tuple)
739 - group_header = (
740 - '<div style="font-size: 1.4em; font-weight: 700; '
741 - 'margin: 18px 0 10px 0;">'
742 - f"{_html.escape(suffix)}"
743 - "</div>\n"
744 - )
960 + csv_dir = Path(args.csv_out_dir) if args.csv_out_dir else None
961 + if csv_dir:
962 + csv_dir.mkdir(parents=True, exist_ok=True)
963 +
964 + excel_path = args.excel_out or "perf_comparison.xlsx"
965 + with pd.ExcelWriter(excel_path, engine="openpyxl") as xw:
966 + # ---- Environment sheet (first) ----
967 + env_sheet = _sanitize_sheet_name("Environment")
968 + env_df = _load_env_df_for_inputs(args, files)
969 + if env_df is None or env_df.empty:
970 + pd.DataFrame(
971 + [
972 + {
973 + "Section": "Environment",
974 + "Key": "vllm_env.txt",
975 + "Value": "NOT FOUND (or empty)",
976 + }
977 + ]
978 + ).to_excel(xw, sheet_name=env_sheet, index=False)
979 + else:
980 + env_df.to_excel(xw, sheet_name=env_sheet, index=False)
981 + with open("perf_comparison.html", "w", encoding="utf-8") as main_fh:
982 + main_fh.write('<meta charset="utf-8">\n')
983 + for gkey in group_keys:
984 + gkey_tuple = normalize_group_key(gkey)
985 + suffix = build_group_suffix(group_cols_canonical, gkey_tuple)
986 + sub_path = group_filename(gkey_tuple)
987 + group_header = (
988 + '<div style="font-size: 1.4em; font-weight: 700; '
989 + 'margin: 18px 0 10px 0;">'
990 + f"{_html.escape(suffix)}"
991 + "</div>\n"
992 + )
993
746 - main_fh.write(group_header)
747 - with open(sub_path, "w", encoding="utf-8") as sub_fh:
748 - sub_fh.write('<meta charset="utf-8">\n')
749 - sub_fh.write(group_header)
750 - tput_group_df = None
751 - ttft_group_df = None
752 - tpot_group_df = None
753 - conc_col = args.xaxis
754 -
755 - for metric_label in plan.data_cols:
756 - gb = metric_groupbys[metric_label]
757 - df_sorted, raw_data_cols = metric_cache[metric_label]
758 -
759 - try:
760 - group_df = gb.get_group(gkey)
761 - except KeyError:
762 - missing = (
763 - '<div style="font-size: 1.1em; font-weight: 600; '
764 - 'margin: 10px 0;">'
765 - f"{_html.escape(metric_label)} — missing for this group"
766 - "</div>\n"
994 + main_fh.write(group_header)
995 +
996 + sheet = _group_to_sheet_base(group_cols_canonical, gkey_tuple)
997 + sheet_base = sheet
998 + dedup_i = 1
999 + while sheet in xw.sheets:
1000 + dedup_i += 1
1001 + sheet = _sanitize_sheet_name(f"{sheet_base}_{dedup_i}")
1002 +
1003 + excel_blocks: list[tuple[str, pd.DataFrame]] = []
1004 +
1005 + with open(sub_path, "w", encoding="utf-8") as sub_fh:
1006 + sub_fh.write('<meta charset="utf-8">\n')
1007 + sub_fh.write(group_header)
1008 + tput_group_df = None
1009 + ttft_group_df = None
1010 + tpot_group_df = None
1011 + conc_col = args.xaxis
1012 +
1013 + for metric_label in plan.data_cols:
1014 + gb = metric_groupbys[metric_label]
1015 + df_sorted, raw_data_cols = metric_cache[metric_label]
1016 +
1017 + try:
1018 + group_df = gb.get_group(gkey)
1019 + except KeyError:
1020 + missing = (
1021 + '<div style="font-size: 1.1em; font-weight: 600; '
1022 + 'margin: 10px 0;">'
1023 + f"{_html.escape(metric_label)} — missing for this group"
1024 + "</div>\n"
1025 + )
1026 + main_fh.write(missing)
1027 + sub_fh.write(missing)
1028 + continue
1029 +
1030 + if conc_col not in group_df.columns:
1031 + conc_col = _find_concurrency_col(group_df)
1032 +
1033 + mn = metric_label.lower().strip()
1034 + if "tok/s" in mn:
1035 + tput_group_df = group_df
1036 + elif "ttft" in mn:
1037 + ttft_group_df = group_df
1038 + elif mn in ("p99", "median") or "tpot" in mn:
1039 + tpot_group_df = group_df
1040 +
1041 + display_group = group_df.drop(
1042 + columns=group_cols_canonical, errors="ignore"
1043 )
1044
769 - main_fh.write(missing)
770 - sub_fh.write(missing)
771 - continue
772 -
773 - if conc_col not in group_df.columns:
774 - conc_col = _find_concurrency_col(group_df)
775 -
776 - mn = metric_label.lower().strip()
777 - if "tok/s" in mn:
778 - tput_group_df = group_df
779 - elif "ttft" in mn:
780 - ttft_group_df = group_df
781 - elif mn in ("p99", "median") or "tpot" in mn:
782 - tpot_group_df = group_df
783 -
784 - display_group = group_df.drop(
785 - columns=group_cols_canonical, errors="ignore"
786 - )
1045 + html = render_metric_table_html(
1046 + display_group, metric_label, suffix, args
1047 + )
1048 + main_fh.write(html)
1049 + sub_fh.write(html)
1050 +
1051 + maybe_write_plot(
1052 + main_fh,
1053 + sub_fh,
1054 + group_df=group_df,
1055 + raw_data_cols=raw_data_cols,
1056 + metric_label=metric_label,
1057 + y_axis_col=y_axis_col,
1058 + args=args,
1059 + )
1060
788 - html = render_metric_table_html(
789 - display_group, metric_label, suffix, args
1061 + excel_blocks.append(
1062 + (metric_label, display_group.reset_index(drop=True))
1063 + )
1064 + if csv_dir:
1065 + fn = _safe_filename(
1066 + f"{sheet}__{metric_label}".replace(" ", "_").replace(
1067 + "/", "_"
1068 + )
1069 + )
1070 + display_group.to_csv(csv_dir / f"{fn}.csv", index=False)
1071 +
1072 + summary_html = build_valid_max_concurrency_summary_html(
1073 + tput_group_df=tput_group_df,
1074 + ttft_group_df=ttft_group_df,
1075 + tpot_group_df=tpot_group_df,
1076 + conc_col=conc_col,
1077 + args=args,
1078 )
791 - main_fh.write(html)
792 - sub_fh.write(html)
793 -
794 - maybe_write_plot(
795 - main_fh,
796 - sub_fh,
797 - group_df=group_df,
798 - raw_data_cols=raw_data_cols,
799 - metric_label=metric_label,
800 - y_axis_col=y_axis_col,
1079 + if summary_html:
1080 + main_fh.write(summary_html)
1081 + sub_fh.write(summary_html)
1082 +
1083 + summary_df = build_valid_max_concurrency_summary_df(
1084 + tput_group_df=tput_group_df,
1085 + ttft_group_df=ttft_group_df,
1086 + tpot_group_df=tpot_group_df,
1087 + conc_col=conc_col,
1088 args=args,
1089 )
1090 + if summary_df is not None:
1091 + excel_blocks.append(
1092 + ("Valid Max Concurrency Summary", summary_df)
1093 + )
1094 + if csv_dir:
1095 + fn = _safe_filename(
1096 + f"{sheet}__Valid_Max_Concurrency_Summary"
1097 + )
1098 + summary_df.to_csv(csv_dir / f"{fn}.csv", index=False)
1099
804 - summary_html = build_valid_max_concurrency_summary_html(
805 - tput_group_df=tput_group_df,
806 - ttft_group_df=ttft_group_df,
807 - tpot_group_df=tpot_group_df,
808 - conc_col=conc_col,
809 - args=args,
810 - )
811 - if summary_html:
812 - main_fh.write(summary_html)
813 - sub_fh.write(summary_html)
1100 + _write_tables_to_excel_sheet(xw, sheet, excel_blocks)
1101 +
1102 + print(f"Wrote Excel: {excel_path}")
1103 + if csv_dir:
1104 + print(f"Wrote CSVs under: {csv_dir}")
1105
1106
1107 def main():
.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
+87 -46
@@ -1,6 +1,4 @@
1 #!/bin/bash
2 -
3 -# This script should be run inside the CI process
2 # This script assumes that we are already inside the vllm/ directory
3 # Benchmarking results will be available inside vllm/benchmarks/results/
4
@@ -9,6 +7,11 @@
7 set -x
8 set -o pipefail
9
10 +# Environment-driven debug controls (like ON_CPU=1)
11 +DRY_RUN="${DRY_RUN:-0}"
12 +MODEL_FILTER="${MODEL_FILTER:-}"
13 +DTYPE_FILTER="${DTYPE_FILTER:-}"
14 +
15 check_gpus() {
16 if command -v nvidia-smi; then
17 # check the number of GPUs and GPU type.
@@ -112,13 +115,12 @@ json2envs() {
115 }
116
117 wait_for_server() {
115 - # wait for vllm server to start
116 - # return 1 if vllm server crashes
118 local timeout_val="1200"
119 timeout "$timeout_val" bash -c '
119 - until curl -X POST localhost:8000/v1/completions; do
120 + until curl -sf http://localhost:8000/v1/models >/dev/null; do
121 sleep 1
121 - done' && return 0 || return 1
122 + done
123 + '
124 }
125
126 kill_processes_launched_by_current_bash() {
@@ -252,37 +254,16 @@ run_benchmark_tests() {
254 done
255 }
256
255 -run_latency_tests() {
256 - run_benchmark_tests "latency" "$1"
257 -}
258 -
259 -run_startup_tests() {
260 - run_benchmark_tests "startup" "$1"
261 -}
262 -
263 -run_throughput_tests() {
264 - run_benchmark_tests "throughput" "$1"
265 -}
266 -
267 -run_serving_tests() {
268 - # run serving tests using `vllm bench serve` command
269 - # $1: a json file specifying serving test cases
270 - #
271 - # Supported JSON formats:
272 - # 1) Plain format: top-level array
273 - # [ { "test_name": "...", "server_parameters": {...}, ... }, ... ]
274 - #
275 - # 2) Default parameters field + plain format tests
276 - # {
277 - # "defaults": { ... },
278 - # "tests": [ { "test_name": "...", "server_parameters": {...}, ... }, ... ]
279 - # }
280 -
281 - local serving_test_file
282 - serving_test_file=$1
257 +run_latency_tests() { run_benchmark_tests "latency" "$1"; }
258 +run_startup_tests() { run_benchmark_tests "startup" "$1"; }
259 +run_throughput_tests() { run_benchmark_tests "throughput" "$1"; }
260
284 - # Iterate over serving tests
285 - jq -c '
261 +merge_serving_tests_stream() {
262 + # Emit merged serving test objects, optionally filtered by MODEL_FILTER/DTYPE_FILTER in DRY_RUN mode.
263 + # This helper does NOT modify JSON; it only filters the stream in dry-run mode.
264 + local serving_test_file="$1"
265 + # shellcheck disable=SC2016
266 + local merged='
267 if type == "array" then
268 # Plain format: test cases array
269 .[]
@@ -304,7 +285,50 @@ run_serving_tests() {
285 else
286 error("Unsupported serving test file format: must be array or object with .tests")
287 end
307 - ' "$serving_test_file" | while read -r params; do
288 + '
289 +
290 + jq -c "$merged" "$serving_test_file" | \
291 + if [[ "${DRY_RUN:-0}" == "1" && ( "${MODEL_FILTER}${DTYPE_FILTER}" != "" ) ]]; then
292 + jq -c --arg model "$MODEL_FILTER" --arg dtype "$DTYPE_FILTER" '
293 + select((($model|length)==0)
294 + or ((.server_parameters.model // "") == $model)
295 + or ((.client_parameters.model // "") == $model))
296 + | select((($dtype|length)==0) or ((.server_parameters.dtype // "") == $dtype))
297 + '
298 + else
299 + cat
300 + fi
301 +}
302 +
303 +run_serving_tests() {
304 + # run serving tests using `vllm bench serve` command
305 + # $1: a json file specifying serving test cases
306 + #
307 + # Supported JSON formats:
308 + # 1) Plain format: top-level array
309 + # [ { "test_name": "...", "server_parameters": {...}, ... }, ... ]
310 + #
311 + # 2) Default parameters field + plain format tests
312 + # {
313 + # "defaults": { ... },
314 + # "tests": [ { "test_name": "...", "server_parameters": {...}, ... }, ... ]
315 + # }
316 +
317 + local serving_test_file
318 + serving_test_file=$1
319 +
320 + # In dry-run mode, if filters are provided but no tests match, fail fast.
321 + if [[ "${DRY_RUN:-0}" == "1" && ( "${MODEL_FILTER}${DTYPE_FILTER}" != "" ) ]]; then
322 + local count
323 + count=$(merge_serving_tests_stream "$serving_test_file" | wc -l | tr -d ' ')
324 + if [[ "$count" -eq 0 ]]; then
325 + echo "No matching serving tests found in $serving_test_file for model='$MODEL_FILTER' dtype='$DTYPE_FILTER'." >&2
326 + return 0
327 + fi
328 + fi
329 +
330 + # Iterate over serving tests (merged + optional filtered stream)
331 + merge_serving_tests_stream "$serving_test_file" | while read -r params; do
332 # get the test name, and append the GPU type back to it.
333 test_name=$(echo "$params" | jq -r '.test_name')
334 if [[ ! "$test_name" =~ ^serving_ ]]; then
@@ -373,7 +397,7 @@ run_serving_tests() {
397 echo "Server command: $server_command"
398 # support remote vllm server
399 client_remote_args=""
376 - if [[ -z "${REMOTE_HOST}" ]]; then
400 + if [[ -z "${REMOTE_HOST}" && "${DRY_RUN:-0}" != "1" ]]; then
401 bash -c "$server_command" &
402 server_pid=$!
403 # wait until the server is alive
@@ -384,6 +408,9 @@ run_serving_tests() {
408 echo ""
409 echo "vLLM failed to start within the timeout period."
410 fi
411 + elif [[ "${DRY_RUN:-0}" == "1" ]]; then
412 + # dry-run: don't start server
413 + echo "Dry Run."
414 else
415 server_command="Using Remote Server $REMOTE_HOST $REMOTE_PORT"
416 if [[ ${REMOTE_PORT} ]]; then
@@ -402,9 +429,7 @@ run_serving_tests() {
429 for qps in $qps_list; do
430 # remove the surrounding single quote from qps
431 if [[ "$qps" == *"inf"* ]]; then
405 - echo "qps was $qps"
432 qps="inf"
407 - echo "now qps is $qps"
433 fi
434
435 # iterate over different max_concurrency
@@ -425,7 +450,9 @@ run_serving_tests() {
450 echo "Running test case $test_name with qps $qps"
451 echo "Client command: $client_command"
452
428 - bash -c "$client_command"
453 + if [[ "${DRY_RUN:-0}" != "1" ]]; then
454 + bash -c "$client_command"
455 + fi
456
457 # record the benchmarking commands
458 jq_output=$(jq -n \
@@ -443,12 +470,15 @@ run_serving_tests() {
470 done
471
472 # clean up
446 - kill -9 $server_pid
447 - kill_gpu_processes
473 + if [[ "${DRY_RUN:-0}" != "1" ]]; then
474 + kill -9 $server_pid
475 + kill_gpu_processes
476 + fi
477 done
478 }
479
480 main() {
481 +
482 local ARCH
483 ARCH=''
484 if [[ "$ON_CPU" == "1" ]]; then
@@ -458,7 +488,13 @@ main() {
488 check_gpus
489 ARCH="$arch_suffix"
490 fi
461 - check_hf_token
491 +
492 + # DRY_RUN does not execute vLLM; do not require HF_TOKEN.
493 + if [[ "${DRY_RUN:-0}" != "1" ]]; then
494 + check_hf_token
495 + else
496 + echo "DRY_RUN=1 -> skip HF_TOKEN validation"
497 + fi
498
499 # dependencies
500 (which wget && which curl) || (apt-get update && apt-get install -y wget curl)
@@ -479,11 +515,16 @@ main() {
515
516 # dump vllm info via vllm collect-env
517 env_output=$(vllm collect-env)
482 -
518 echo "$env_output" >"$RESULTS_FOLDER/vllm_env.txt"
519
520 # benchmarking
486 - run_serving_tests $QUICK_BENCHMARK_ROOT/tests/"${SERVING_JSON:-serving-tests$ARCH.json}"
521 + run_serving_tests $QUICK_BENCHMARK_ROOT/tests/"${SERVING_JSON:-serving-tests$ARCH.json}" || exit $?
522 +
523 + if [[ "${DRY_RUN:-0}" == "1" ]]; then
524 + echo "DRY_RUN=1 -> skip latency/startup/throughput suites"
525 + exit 0
526 + fi
527 +
528 run_latency_tests $QUICK_BENCHMARK_ROOT/tests/"${LATENCY_JSON:-latency-tests$ARCH.json}"
529 run_startup_tests $QUICK_BENCHMARK_ROOT/tests/"${STARTUP_JSON:-startup-tests$ARCH.json}"
530 run_throughput_tests $QUICK_BENCHMARK_ROOT/tests/"${THROUGHPUT_JSON:-throughput-tests$ARCH.json}"
.buildkite/performance-benchmarks/tests/serving-tests-cpu-embed.json new
+41
@@ -0,0 +1,41 @@
1 +{
2 + "defaults": {
3 + "qps_list": [
4 + "inf"
5 + ],
6 + "max_concurrency_list": [
7 + 32,
8 + 64,
9 + 128
10 + ],
11 + "server_environment_variables": {
12 + "VLLM_RPC_TIMEOUT": 100000,
13 + "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
14 + "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
15 + "VLLM_CPU_SGL_KERNEL": 1,
16 + "VLLM_CPU_KVCACHE_SPACE": 40
17 + },
18 + "server_parameters": {
19 + "dtype": "bfloat16",
20 + "model": "jinaai/jina-embeddings-v3",
21 + "trust_remote_code": ""
22 + },
23 + "client_parameters": {
24 + "model": "jinaai/jina-embeddings-v3",
25 + "backend": "openai-embeddings",
26 + "endpoint": "/v1/embeddings",
27 + "dataset_name": "sharegpt",
28 + "dataset_path": "ShareGPT_V3_unfiltered_cleaned_split.json",
29 + "num_prompts": 200
30 + }
31 + },
32 + "tests": [
33 + {
34 + "test_name": "serving_jina_embed_v3_tp1_sharegpt",
35 + "server_parameters": {
36 + "tensor_parallel_size": 1
37 + },
38 + "client_parameters": {}
39 + }
40 + ]
41 +}
.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json new
+283
@@ -0,0 +1,283 @@
1 +{
2 + "defaults": {
3 + "qps_list": [
4 + "inf"
5 + ],
6 + "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
7 + "server_environment_variables": {
8 + "VLLM_RPC_TIMEOUT": 100000,
9 + "VLLM_ALLOW_LONG_MAX_MODEL_LEN": 1,
10 + "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120,
11 + "VLLM_CPU_SGL_KERNEL": 1,
12 + "VLLM_CPU_KVCACHE_SPACE": 40
13 + },
14 + "server_parameters": {
15 + "model": "meta-llama/Llama-3.1-8B-Instruct",
16 + "tensor_parallel_size": 1,
17 + "dtype": "bfloat16",
18 + "distributed_executor_backend": "mp",
19 + "block_size": 128,
20 + "trust_remote_code": "",
21 + "disable_log_stats": "",
22 + "max_num_batched_tokens": 2048,
23 + "max_num_seqs": 256
24 + },
25 + "client_parameters": {
26 + "model": "meta-llama/Llama-3.1-8B-Instruct",
27 + "backend": "vllm",
28 + "ignore-eos": "",
29 + "num_prompts": 200
30 + }
31 + },
32 + "tests": [
33 + {
34 + "test_name": "serving_llama8B_tp1_sharegpt",
35 + "server_parameters": {
36 + "tensor_parallel_size": 1
37 + },
38 + "client_parameters": {
39 + "dataset_name": "sharegpt",
40 + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
41 + }
42 + },
43 + {
44 + "test_name": "serving_llama8B_tp2_sharegpt",
45 + "server_parameters": {
46 + "tensor_parallel_size": 2
47 + },
48 + "client_parameters": {
49 + "dataset_name": "sharegpt",
50 + "dataset_path": "./ShareGPT_V3_unfiltered_cleaned_split.json"
51 + }
52 + },
53 + {
54 + "test_name": "serving_llama8B_tp1_random_128_128",
55 + "server_parameters": {
56 + "tensor_parallel_size": 1
57 + },
58 + "client_parameters": {
59 + "dataset_name": "random",
60 + "random-input-len": 128,
61 + "random-output-len": 128
62 + }
63 + },
64 + {
65 + "test_name": "serving_llama8B_tp2_random_128_128",
66 + "server_parameters": {
67 + "tensor_parallel_size": 2
68 + },
69 + "client_parameters": {
70 + "dataset_name": "random",
71 + "random-input-len": 128,
72 + "random-output-len": 128
73 + }
74 + },
75 + {
76 + "test_name": "serving_llama8B_tp4_random_128_128",
77 + "server_parameters": {
78 + "tensor_parallel_size": 4
79 + },
80 + "client_parameters": {
81 + "dataset_name": "random",
82 + "random-input-len": 128,
83 + "random-output-len": 128
84 + }
85 + },
86 + {
87 + "test_name": "serving_llama8B_tp1_random_128_2048",
88 + "server_parameters": {
89 + "tensor_parallel_size": 1
90 + },
91 + "client_parameters": {
92 + "dataset_name": "random",
93 + "random-input-len": 128,
94 + "random-output-len": 2048
95 + }
96 + },
97 + {
98 + "test_name": "serving_llama8B_tp2_random_128_2048",
99 + "server_parameters": {
100 + "tensor_parallel_size": 2
101 + },
102 + "client_parameters": {
103 + "dataset_name": "random",
104 + "random-input-len": 128,
105 + "random-output-len": 2048
106 + }
107 + },
108 + {
109 + "test_name": "serving_llama8B_tp4_random_128_2048",
110 + "server_parameters": {
111 + "tensor_parallel_size": 4
112 + },
113 + "client_parameters": {
114 + "dataset_name": "random",
115 + "random-input-len": 128,
116 + "random-output-len": 2048
117 + }
118 + },
119 + {
120 + "test_name": "serving_llama8B_tp1_random_2048_128",
121 + "server_parameters": {
122 + "tensor_parallel_size": 1
123 + },
124 + "client_parameters": {
125 + "dataset_name": "random",
126 + "random-input-len": 2048,
127 + "random-output-len": 128
128 + }
129 + },
130 + {
131 + "test_name": "serving_llama8B_tp2_random_2048_128",
132 + "server_parameters": {
133 + "tensor_parallel_size": 2
134 + },
135 + "client_parameters": {
136 + "dataset_name": "random",
137 + "random-input-len": 2048,
138 + "random-output-len": 128
139 + }
140 + },
141 + {
142 + "test_name": "serving_llama8B_tp4_random_2048_128",
143 + "server_parameters": {
144 + "tensor_parallel_size": 4
145 + },
146 + "client_parameters": {
147 + "dataset_name": "random",
148 + "random-input-len": 2048,
149 + "random-output-len": 128
150 + }
151 + },
152 + {
153 + "test_name": "serving_llama8B_int4_tp1_random_128_128",
154 + "server_parameters": {
155 + "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
156 + "tensor_parallel_size": 1
157 + },
158 + "client_parameters": {
159 + "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
160 + "dataset_name": "random",
161 + "random-input-len": 128,
162 + "random-output-len": 128
163 + }
164 + },
165 + {
166 + "test_name": "serving_llama8B_int4_tp2_random_128_128",
167 + "server_parameters": {
168 + "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
169 + "tensor_parallel_size": 2
170 + },
171 + "client_parameters": {
172 + "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
173 + "dataset_name": "random",
174 + "random-input-len": 128,
175 + "random-output-len": 128
176 + }
177 + },
178 + {
179 + "test_name": "serving_llama8B_int4_tp4_random_128_128",
180 + "server_parameters": {
181 + "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
182 + "tensor_parallel_size": 4
183 + },
184 + "client_parameters": {
185 + "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
186 + "dataset_name": "random",
187 + "random-input-len": 128,
188 + "random-output-len": 128
189 + }
190 + },
191 + {
192 + "test_name": "serving_llama3B_tp1_random_128_128",
193 + "server_parameters": {
194 + "model": "meta-llama/Llama-3.2-3B-Instruct",
195 + "tensor_parallel_size": 1
196 + },
197 + "client_parameters": {
198 + "model": "meta-llama/Llama-3.2-3B-Instruct",
199 + "dataset_name": "random",
200 + "random-input-len": 128,
201 + "random-output-len": 128
202 + }
203 + },
204 + {
205 + "test_name": "serving_granite2B_tp1_random_128_128",
206 + "server_parameters": {
207 + "model": "ibm-granite/granite-3.2-2b-instruct",
208 + "tensor_parallel_size": 1
209 + },
210 + "client_parameters": {
211 + "model": "ibm-granite/granite-3.2-2b-instruct",
212 + "dataset_name": "random",
213 + "random-input-len": 128,
214 + "random-output-len": 128
215 + }
216 + },
217 + {
218 + "test_name": "serving_qwen1.7B_tp1_random_128_128",
219 + "server_parameters": {
220 + "model": "Qwen/Qwen3-1.7B",
221 + "tensor_parallel_size": 1
222 + },
223 + "client_parameters": {
224 + "model": "Qwen/Qwen3-1.7B",
225 + "dataset_name": "random",
226 + "random-input-len": 128,
227 + "random-output-len": 128
228 + }
229 + },
230 + {
231 + "test_name": "serving_qwen4B_tp1_random_128_128",
232 + "server_parameters": {
233 + "model": "Qwen/Qwen3-4B",
234 + "tensor_parallel_size": 1
235 + },
236 + "client_parameters": {
237 + "model": "Qwen/Qwen3-4B",
238 + "dataset_name": "random",
239 + "random-input-len": 128,
240 + "random-output-len": 128
241 + }
242 + },
243 + {
244 + "test_name": "serving_qwen8B_tp1_random_128_128",
245 + "server_parameters": {
246 + "model": "Qwen/Qwen3-8B",
247 + "tensor_parallel_size": 1
248 + },
249 + "client_parameters": {
250 + "model": "Qwen/Qwen3-8B",
251 + "dataset_name": "random",
252 + "random-input-len": 128,
253 + "random-output-len": 128
254 + }
255 + },
256 + {
257 + "test_name": "serving_glm9B_tp1_random_128_128",
258 + "server_parameters": {
259 + "model": "zai-org/glm-4-9b-hf",
260 + "tensor_parallel_size": 1
261 + },
262 + "client_parameters": {
263 + "model": "zai-org/glm-4-9b-hf",
264 + "dataset_name": "random",
265 + "random-input-len": 128,
266 + "random-output-len": 128
267 + }
268 + },
269 + {
270 + "test_name": "serving_gemma7B_tp1_random_128_128",
271 + "server_parameters": {
272 + "model": "google/gemma-7b",
273 + "tensor_parallel_size": 1
274 + },
275 + "client_parameters": {
276 + "model": "google/gemma-7b",
277 + "dataset_name": "random",
278 + "random-input-len": 128,
279 + "random-output-len": 128
280 + }
281 + }
282 + ]
283 +}
.buildkite/performance-benchmarks/tests/serving-tests-cpu.json
-130
@@ -148,136 +148,6 @@
148 "random-input-len": 2048,
149 "random-output-len": 128
150 }
151 - },
152 - {
153 - "test_name": "serving_llama8B_int4_tp1_random_128_128",
154 - "server_parameters": {
155 - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
156 - "tensor_parallel_size": 1
157 - },
158 - "client_parameters": {
159 - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
160 - "dataset_name": "random",
161 - "random-input-len": 128,
162 - "random-output-len": 128
163 - }
164 - },
165 - {
166 - "test_name": "serving_llama8B_int4_tp2_random_128_128",
167 - "server_parameters": {
168 - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
169 - "tensor_parallel_size": 2
170 - },
171 - "client_parameters": {
172 - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
173 - "dataset_name": "random",
174 - "random-input-len": 128,
175 - "random-output-len": 128
176 - }
177 - },
178 - {
179 - "test_name": "serving_llama8B_int4_tp4_random_128_128",
180 - "server_parameters": {
181 - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
182 - "tensor_parallel_size": 4
183 - },
184 - "client_parameters": {
185 - "model": "hugging-quants/Meta-Llama-3.1-8B-Instruct-AWQ-INT4",
186 - "dataset_name": "random",
187 - "random-input-len": 128,
188 - "random-output-len": 128
189 - }
190 - },
191 - {
192 - "test_name": "serving_llama3B_tp1_random_128_128",
193 - "server_parameters": {
194 - "model": "meta-llama/Llama-3.2-3B-Instruct",
195 - "tensor_parallel_size": 1
196 - },
197 - "client_parameters": {
198 - "model": "meta-llama/Llama-3.2-3B-Instruct",
199 - "dataset_name": "random",
200 - "random-input-len": 128,
201 - "random-output-len": 128
202 - }
203 - },
204 - {
205 - "test_name": "serving_granite2B_tp1_random_128_128",
206 - "server_parameters": {
207 - "model": "ibm-granite/granite-3.2-2b-instruct",
208 - "tensor_parallel_size": 1
209 - },
210 - "client_parameters": {
211 - "model": "ibm-granite/granite-3.2-2b-instruct",
212 - "dataset_name": "random",
213 - "random-input-len": 128,
214 - "random-output-len": 128
215 - }
216 - },
217 - {
218 - "test_name": "serving_qwen1.7B_tp1_random_128_128",
219 - "server_parameters": {
220 - "model": "Qwen/Qwen3-1.7B",
221 - "tensor_parallel_size": 1
222 - },
223 - "client_parameters": {
224 - "model": "Qwen/Qwen3-1.7B",
225 - "dataset_name": "random",
226 - "random-input-len": 128,
227 - "random-output-len": 128
228 - }
229 - },
230 - {
231 - "test_name": "serving_qwen4B_tp1_random_128_128",
232 - "server_parameters": {
233 - "model": "Qwen/Qwen3-4B",
234 - "tensor_parallel_size": 1
235 - },
236 - "client_parameters": {
237 - "model": "Qwen/Qwen3-4B",
238 - "dataset_name": "random",
239 - "random-input-len": 128,
240 - "random-output-len": 128
241 - }
242 - },
243 - {
244 - "test_name": "serving_qwen8B_tp1_random_128_128",
245 - "server_parameters": {
246 - "model": "Qwen/Qwen3-8B",
247 - "tensor_parallel_size": 1
248 - },
249 - "client_parameters": {
250 - "model": "Qwen/Qwen3-8B",
251 - "dataset_name": "random",
252 - "random-input-len": 128,
253 - "random-output-len": 128
254 - }
255 - },
256 - {
257 - "test_name": "serving_glm9B_tp1_random_128_128",
258 - "server_parameters": {
259 - "model": "zai-org/glm-4-9b-hf",
260 - "tensor_parallel_size": 1
261 - },
262 - "client_parameters": {
263 - "model": "zai-org/glm-4-9b-hf",
264 - "dataset_name": "random",
265 - "random-input-len": 128,
266 - "random-output-len": 128
267 - }
268 - },
269 - {
270 - "test_name": "serving_gemma7B_tp1_random_128_128",
271 - "server_parameters": {
272 - "model": "google/gemma-7b",
273 - "tensor_parallel_size": 1
274 - },
275 - "client_parameters": {
276 - "model": "google/gemma-7b",
277 - "dataset_name": "random",
278 - "random-input-len": 128,
279 - "random-output-len": 128
280 - }
151 }
152 ]
153 }
docs/getting_started/installation/cpu.md
+23 -1
@@ -176,7 +176,7 @@ For the full and up-to-date list of models validated on CPU platforms, please se
176
177 ### How to find benchmark configuration examples for supported CPU models?
178
179 -For any model listed under [Supported Models on CPU](../../models/hardware_supported_models/cpu.md), optimized runtime configurations are provided in the vLLM Benchmark Suite’s CPU test cases, defined in [cpu test cases](../../../.buildkite/performance-benchmarks/tests/serving-tests-cpu.json)
179 +For any model listed under [Supported Models on CPU](../../models/hardware_supported_models/cpu.md), optimized runtime configurations are provided in the vLLM Benchmark Suite’s CPU test cases, defined in cpu test cases as serving-tests-cpu.json. Full test cases for Text-only models, Multi-Modal models and Embedded models are in cpu Text-Only test cases as serving-tests-cpu-text.json, cpu Multi-Modal test cases as serving-tests-cpu-multimodal.json and cpu Embedded test cases as serving-tests-cpu-embed.json.
180 For details on how these optimized configurations are determined, see: [performance-benchmark-details](../../../.buildkite/performance-benchmarks/README.md#performance-benchmark-details).
181 To benchmark the supported models using these optimized settings, follow the steps in [running vLLM Benchmark Suite manually](../../benchmarking/dashboard.md#manually-trigger-the-benchmark) and run the Benchmark Suite on a CPU environment.
182
@@ -199,6 +199,28 @@ lscpu | grep "NUMA node(s):" | awk '{print $3}'
199 For performance reference, users may also consult the [vLLM Performance Dashboard](https://hud.pytorch.org/benchmark/llms?repoName=vllm-project%2Fvllm&deviceName=cpu)
200 , which publishes default-model CPU results produced using the same Benchmark Suite.
201
202 +#### Dry-Run
203 +
204 +For users only need to get the optimized runtime configurations without running benchmark, a Dry-Run mode is provided.
205 +By passing an environment variable DRY_RUN=1 with run-performance-benchmarks.sh,
206 +all commands will be generated under `./benchmark/results/`.
207 +
208 +```bash
209 +ON_CPU=1 DRY_RUN=1 bash .buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
210 +```
211 +
212 +By providing different JSON file, users can get runtime configurations for different models such as Embedded Models.
213 +
214 +```bash
215 +ON_CPU=1 SERVING_JSON=serving-tests-cpu-embed.json DRY_RUN=1 bash .buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
216 +```
217 +
218 +By providing MODEL_FILTER and DTYPE_FILTER, only commands for related model ID and Data Type will be generated.
219 +
220 +```bash
221 +ON_CPU=1 SERVING_JSON=serving-tests-cpu-text.json DRY_RUN=1 MODEL_FILTER=meta-llama/Llama-3.1-8B-Instruct DTYPE_FILTER=bfloat16 bash .buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
222 +```
223 +
224 ### How to decide `VLLM_CPU_OMP_THREADS_BIND`?
225
226 - Default `auto` thread-binding is recommended for most cases. Ideally, each OpenMP thread will be bound to a dedicated physical core respectively, threads of each rank will be bound to the same NUMA node respectively, and 1 CPU per rank will be reserved for other vLLM components when `world_size > 1`. If you have any performance problems or unexpected binding behaviours, please try to bind threads as following.