A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 20:30 UTC
more models for vLLM Benchmark Suite (#35086)
Signed-off-by: louie-tsai <louie.tsai@intel.com>
Louie Tsai committed
Mar 11, 2026 at 20:36 UTC
17852aa503bf8b0d0d996bf7ec7f3388790ac50e
8 files changed
+800
-119
.buildkite/performance-benchmarks/scripts/compare-json-results.py
+301
-90
@@ -7,12 +7,12 @@ import argparse
7
import html as _html
8
import json
9
import os
10
+from contextlib import nullcontext
11
from dataclasses import dataclass
12
from importlib import util
13
from pathlib import Path
14
15
import pandas as pd
15
-import regex as re
16
17
pd.options.display.float_format = "{:.2f}".format
18
plotly_found = util.find_spec("plotly.express") is not None
@@ -33,6 +33,45 @@ pd.set_option("display.precision", 2)
33
pd.set_option("display.float_format", lambda x: f"{x:.2f}")
34
35
36
+# -----------------------------
37
+# Concurrency normalization (NEW, small)
38
+# -----------------------------
39
+def _find_concurrency_col(df: pd.DataFrame) -> str:
40
+ for c in [
41
+ "# of max concurrency.",
42
+ "# of max concurrency",
43
+ "Max Concurrency",
44
+ "max_concurrency",
45
+ "Concurrency",
46
+ ]:
47
+ if c in df.columns:
48
+ return c
49
+
50
+ for c in df.columns:
51
+ if "concurr" in str(c).lower():
52
+ s = df[c]
53
+ if s.dtype.kind in "iu" and s.nunique() > 1 and s.min() >= 1:
54
+ return c
55
+
56
+ raise ValueError(
57
+ "Cannot infer concurrency column. "
58
+ "Please rename the column to one of the known names "
59
+ "or add an explicit override (e.g., --concurrency-col)."
60
+ )
61
+
62
+
63
+def _normalize_concurrency_in_df(
64
+ df: pd.DataFrame, canonical: str = "# of max concurrency."
65
+) -> pd.DataFrame:
66
+ if canonical in df.columns:
67
+ return df
68
+ detected = _find_concurrency_col(df)
69
+ if detected in df.columns and detected != canonical:
70
+ return df.rename(columns={detected: canonical})
71
+ df[canonical] = pd.NA
72
+ return df
73
+
74
+
75
# -----------------------------
76
# Core data compare
77
# -----------------------------
@@ -52,19 +91,25 @@ def compare_data_columns(
91
- Concat along axis=1 (indexes align), then reset_index so callers can
92
group by columns.
93
- If --debug, add a <file_label>_name column per file.
94
+
95
+ Minimal fix to support different max_concurrency lists across files:
96
+ - normalize concurrency column naming to "# of max concurrency."
97
+ - align on UNION of keys (missing points become NaN)
98
+ - BUGFIX: don't drop throughput rows based on P99/Median presence
99
"""
100
print("\ncompare_data_column:", data_column)
101
102
frames = []
103
raw_data_cols: list[str] = []
60
- compare_frames = []
104
105
+ # Determine key cols after normalizing concurrency
106
cols_per_file: list[set] = []
107
for f in files:
108
try:
109
df_tmp = pd.read_json(f, orient="records")
110
except Exception as err:
111
raise ValueError(f"Failed to read {f}") from err
112
+ df_tmp = _normalize_concurrency_in_df(df_tmp, canonical="# of max concurrency.")
113
cols_per_file.append(set(df_tmp.columns))
114
115
key_cols = [c for c in info_cols if all(c in cset for cset in cols_per_file)]
@@ -75,12 +120,25 @@ def compare_data_columns(
120
"No common key columns found from info_cols across the input files."
121
)
122
78
- meta_added = False
123
+ union_index = None
124
+ metas: list[pd.DataFrame] = []
125
+ staged: list[tuple[str, pd.Series, pd.Series | None]] = []
126
127
for file in files:
128
df = pd.read_json(file, orient="records")
82
-
83
- if drop_column in df.columns:
129
+ df = _normalize_concurrency_in_df(df, canonical="# of max concurrency.")
130
+
131
+ # BUGFIX: only drop rows for latency-like metrics; throughput rows may have
132
+ # NaN in P99/Median columns even if the column exists in the JSON.
133
+ metric_lc = str(data_column).lower()
134
+ is_latency_metric = (
135
+ "ttft" in metric_lc
136
+ or "tpot" in metric_lc
137
+ or "p99" in metric_lc
138
+ or "median" in metric_lc
139
+ or metric_lc.strip() in {"p99", "median"}
140
+ )
141
+ if is_latency_metric and drop_column in df.columns:
142
df = df.dropna(subset=[drop_column], ignore_index=True)
143
144
for c in (
@@ -105,35 +163,61 @@ def compare_data_columns(
163
meta = meta.groupby(level=key_cols, dropna=False).first()
164
165
file_label = "/".join(file.split("/")[:-1]) or os.path.basename(file)
108
- s = df_idx[data_column]
109
- if not s.index.is_unique:
110
- s = s.groupby(level=key_cols, dropna=False).mean()
111
- s.name = file_label
166
113
- if not meta_added:
114
- frames.append(meta)
115
- meta_added = True
167
+ if data_column in df_idx.columns:
168
+ s = df_idx[data_column]
169
+ if not s.index.is_unique:
170
+ s = s.groupby(level=key_cols, dropna=False).mean()
171
+ else:
172
+ # keep NA series to preserve meta keys for union_index
173
+ s = pd.Series(pd.NA, index=meta.index)
174
+ s.name = file_label
175
176
+ name_s = None
177
if debug and name_column in df_idx.columns:
178
name_s = df_idx[name_column]
179
if not name_s.index.is_unique:
180
name_s = name_s.groupby(level=key_cols, dropna=False).first()
181
name_s.name = f"{file_label}_name"
122
- frames.append(name_s)
182
124
- frames.append(s)
183
+ if union_index is None:
184
+ union_index = meta.index
185
+ else:
186
+ union_index = union_index.union(meta.index)
187
+ metas.append(meta)
188
+
189
+ staged.append((file_label, s, name_s))
190
+
191
+ if union_index is None:
192
+ raise ValueError("No data found after loading inputs.")
193
+
194
+ # meta first (union-aligned): build UNION meta across all files
195
+ if metas:
196
+ meta_union = pd.concat(metas, axis=0)
197
+ # Collapse duplicates on the MultiIndex; keep first non-null per column
198
+ meta_union = meta_union.groupby(level=key_cols, dropna=False).first()
199
+ frames.append(meta_union.reindex(union_index))
200
+
201
+ # values + ratios (union-aligned)
202
+ metric_series_aligned: list[pd.Series] = []
203
+ for file_label, s, name_s in staged:
204
+ s_aligned = s.reindex(union_index)
205
+ frames.append(s_aligned)
206
raw_data_cols.append(file_label)
126
- compare_frames.append(s)
207
+ metric_series_aligned.append(s_aligned)
208
+
209
+ if debug and name_s is not None:
210
+ frames.append(name_s.reindex(union_index))
211
128
- if len(compare_frames) >= 2:
129
- base = compare_frames[0]
130
- current = compare_frames[-1]
131
- if "P99" in data_column or "Median" in data_column:
212
+ if len(metric_series_aligned) >= 2:
213
+ base = metric_series_aligned[0]
214
+ current = metric_series_aligned[-1]
215
+ if "P99" in str(data_column) or "Median" in str(data_column):
216
ratio = base / current
217
else:
218
ratio = current / base
219
ratio = ratio.mask(base == 0)
136
- ratio.name = f"Ratio 1 vs {len(compare_frames)}"
220
+ ratio.name = f"Ratio 1 vs {len(metric_series_aligned)}"
221
frames.append(ratio)
222
223
concat_df = pd.concat(frames, axis=1).reset_index(drop=True)
@@ -204,24 +288,10 @@ def split_json_by_tp_pp(
288
# -----------------------------
289
# Styling helpers
290
# -----------------------------
207
-def _find_concurrency_col(df: pd.DataFrame) -> str:
208
- for c in [
209
- "# of max concurrency.",
210
- "# of max concurrency",
211
- "Max Concurrency",
212
- "max_concurrency",
213
- "Concurrency",
214
- ]:
215
- if c in df.columns:
216
- return c
217
- for c in df.columns:
218
- if df[c].dtype.kind in "iu" and df[c].nunique() > 1 and df[c].min() >= 1:
219
- return c
220
- return "# of max concurrency."
221
-
222
-
291
def _highlight_threshold(
224
- df: pd.DataFrame, threshold: float
292
+ df: pd.DataFrame,
293
+ threshold: float,
294
+ slack_pct: float = 0.0,
295
) -> pd.io.formats.style.Styler:
296
conc_col = _find_concurrency_col(df)
297
key_cols = [
@@ -234,12 +304,24 @@ def _highlight_threshold(
304
]
305
conf_cols = [c for c in conf_cols if pd.api.types.is_numeric_dtype(df[c])]
306
237
- return df.style.map(
238
- lambda v: "background-color:#e6ffe6;font-weight:bold;"
239
- if pd.notna(v) and v <= threshold
240
- else "",
241
- subset=conf_cols,
242
- )
307
+ try:
308
+ slack_pct = float(slack_pct or 0.0)
309
+ except Exception:
310
+ slack_pct = 0.0
311
+ slack_limit = threshold * (1.0 + slack_pct / 100.0)
312
+
313
+ def _cell(v):
314
+ if pd.isna(v):
315
+ return ""
316
+ if v <= threshold:
317
+ # Strict SLA
318
+ return "background-color:#e6ffe6;font-weight:bold;"
319
+ if v <= slack_limit:
320
+ # Within slack range
321
+ return "background-color:#ffe5cc;font-weight:bold;"
322
+ return ""
323
+
324
+ return df.style.map(_cell, subset=conf_cols)
325
326
327
def highlight_ratio_columns(styler: pd.io.formats.style.Styler):
@@ -286,11 +368,30 @@ def _sanitize_sheet_name(name: str) -> str:
368
- max 31 chars
369
- cannot contain: : \ / ? * [ ]
370
- cannot be empty
371
+
372
+ NOTE: Use fast, non-regex operations here to avoid the third-party `regex`
373
+ module's compile overhead/edge-cases on some systems.
374
"""
375
name = "sheet" if name is None else str(name)
291
- name = re.sub(r"[:\\/?*\[\]]", "_", name)
376
+
377
+ # Replace illegal characters with underscore.
378
+ trans = str.maketrans(
379
+ {
380
+ ":": "_",
381
+ "\\": "_",
382
+ "/": "_",
383
+ "?": "_",
384
+ "*": "_",
385
+ "[": "_",
386
+ "]": "_",
387
+ }
388
+ )
389
+ name = name.translate(trans)
390
+
391
+ # Strip quotes/spaces and collapse whitespace.
392
name = name.strip().strip("'")
293
- name = re.sub(r"\s+", " ", name)
393
+ name = " ".join(name.split())
394
+
395
if not name:
396
name = "sheet"
397
return name[:31]
@@ -298,30 +399,57 @@ def _sanitize_sheet_name(name: str) -> str:
399
400
def _group_to_sheet_base(group_cols: list[str], gkey_tuple) -> str:
401
d = dict(zip(group_cols, gkey_tuple))
301
- model = d.get("Model", "model")
302
- model_short = str(model).split("/")[-1]
402
+
403
+ # Always keep input/output lengths (these are important).
404
ilen = d.get("Input Len", "")
405
olen = d.get("Output Len", "")
406
lens = f"_{ilen}x{olen}" if ilen != "" and olen != "" else ""
407
+
408
+ # Shorten model name aggressively to make room for lens.
409
+ model = d.get("Model", "model")
410
+ leaf = str(model).split("/")[-1]
411
+
412
+ max_model_len = max(1, 31 - len(lens))
413
+ model_short = leaf[:max_model_len]
414
+
415
return _sanitize_sheet_name(f"{model_short}{lens}")
416
417
418
def _write_tables_to_excel_sheet(
419
writer: pd.ExcelWriter, sheet: str, blocks: list[tuple[str, pd.DataFrame]]
420
):
312
- startrow = 0
421
+ """Write all blocks to a sheet with a single to_excel() call.
422
+
423
+ Pandas+openpyxl can be extremely slow when called many times per sheet.
424
+ We flatten blocks into one table with a 'Section' column to keep structure
425
+ while making Excel generation fast and deterministic.
426
+ """
427
+ if not blocks:
428
+ pd.DataFrame().to_excel(writer, sheet_name=sheet, index=False)
429
+ return
430
+
431
+ combined_parts: list[pd.DataFrame] = []
432
for title, df in blocks:
314
- pd.DataFrame([[title]]).to_excel(
315
- writer, sheet_name=sheet, index=False, header=False, startrow=startrow
316
- )
317
- startrow += 1
318
- df.to_excel(writer, sheet_name=sheet, index=False, startrow=startrow)
319
- startrow += len(df) + 3
433
+ df2 = df.copy()
434
+ # Put the section label as the first column for readability.
435
+ df2.insert(0, "Section", title)
436
+ combined_parts.append(df2)
437
+
438
+ combined = pd.concat(combined_parts, axis=0, ignore_index=True, sort=False)
439
+ combined.to_excel(writer, sheet_name=sheet, index=False)
440
441
442
def _safe_filename(s: str) -> str:
323
- s = re.sub(r"[^\w\-.]+", "_", str(s).strip())
324
- return s[:180] if len(s) > 180 else s
443
+ # Fast path without the third-party `regex` module.
444
+ s = " ".join(str(s).strip().split())
445
+ allowed = []
446
+ for ch in s:
447
+ if ch.isalnum() or ch in "._-":
448
+ allowed.append(ch)
449
+ else:
450
+ allowed.append("_")
451
+ out = "".join(allowed)
452
+ return out[:180] if len(out) > 180 else out
453
454
455
# -----------------------------
@@ -428,7 +556,11 @@ def _config_value_columns(df: pd.DataFrame, conc_col: str) -> list[str]:
556
557
558
def _max_concurrency_ok(
431
- df: pd.DataFrame, conc_col: str, cfg_col: str, threshold: float
559
+ df: pd.DataFrame,
560
+ conc_col: str,
561
+ cfg_col: str,
562
+ threshold: float,
563
+ slack_pct: float = 0.0,
564
):
565
if df is None or conc_col not in df.columns or cfg_col not in df.columns:
566
return pd.NA
@@ -441,7 +573,14 @@ def _max_concurrency_ok(
573
if d.empty:
574
return pd.NA
575
444
- ok = d[d[cfg_col] <= threshold]
576
+ # Accept values up to (1 + slack_pct%) above the SLA.
577
+ try:
578
+ slack_pct = float(slack_pct or 0.0)
579
+ except Exception:
580
+ slack_pct = 0.0
581
+ effective_limit = float(threshold) * (1.0 + slack_pct / 100.0)
582
+
583
+ ok = d[d[cfg_col] <= effective_limit]
584
if ok.empty:
585
return pd.NA
586
@@ -507,15 +646,25 @@ def build_valid_max_concurrency_summary_html(
646
if not cfg_cols:
647
cfg_cols = sorted(set(ttft_cols) | set(tpot_cols) | set(tput_cols), key=str)
648
649
+ # Display SLA ranges in the table header (SLA .. SLA*(1+slack))
650
+ ttft_hi = args.ttft_max_ms * (1.0 + args.ttft_slack_pct / 100.0)
651
+ tpot_hi = args.tpot_max_ms * (1.0 + args.tpot_slack_pct / 100.0)
652
+ ttft_range = f"{args.ttft_max_ms:g}–{ttft_hi:g} ms (+{args.ttft_slack_pct:g}%)"
653
+ tpot_range = f"{args.tpot_max_ms:g}–{tpot_hi:g} ms (+{args.tpot_slack_pct:g}%)"
654
+
655
rows = []
656
for cfg in cfg_cols:
657
ttft_max = (
513
- _max_concurrency_ok(ttft_group_df, conc_col, cfg, args.ttft_max_ms)
658
+ _max_concurrency_ok(
659
+ ttft_group_df, conc_col, cfg, args.ttft_max_ms, args.ttft_slack_pct
660
+ )
661
if ttft_group_df is not None
662
else pd.NA
663
)
664
tpot_max = (
518
- _max_concurrency_ok(tpot_group_df, conc_col, cfg, args.tpot_max_ms)
665
+ _max_concurrency_ok(
666
+ tpot_group_df, conc_col, cfg, args.tpot_max_ms, args.tpot_slack_pct
667
+ )
668
if tpot_group_df is not None
669
else pd.NA
670
)
@@ -544,8 +693,8 @@ def build_valid_max_concurrency_summary_html(
693
rows.append(
694
{
695
"Configuration": cfg,
547
- f"Max {conc_col} (TTFT ≤ {args.ttft_max_ms:g} ms)": ttft_max,
548
- f"Max {conc_col} (TPOT ≤ {args.tpot_max_ms:g} ms)": tpot_max,
696
+ f"Max {conc_col} (TTFT ≤ {ttft_range})": ttft_max,
697
+ f"Max {conc_col} (TPOT ≤ {tpot_range})": tpot_max,
698
f"Max {conc_col} (Both)": both,
699
"Output Tput @ Both (tok/s)": tput_at_both,
700
"TTFT @ Both (ms)": ttft_at_both,
@@ -620,15 +769,24 @@ def build_valid_max_concurrency_summary_df(
769
if not cfg_cols:
770
cfg_cols = sorted(set(ttft_cols) | set(tpot_cols) | set(tput_cols), key=str)
771
772
+ ttft_hi = args.ttft_max_ms * (1.0 + args.ttft_slack_pct / 100.0)
773
+ tpot_hi = args.tpot_max_ms * (1.0 + args.tpot_slack_pct / 100.0)
774
+ ttft_range = f"{args.ttft_max_ms:g}–{ttft_hi:g} ms (+{args.ttft_slack_pct:g}%)"
775
+ tpot_range = f"{args.tpot_max_ms:g}–{tpot_hi:g} ms (+{args.tpot_slack_pct:g}%)"
776
+
777
rows = []
778
for cfg in cfg_cols:
779
ttft_max = (
626
- _max_concurrency_ok(ttft_group_df, conc_col, cfg, args.ttft_max_ms)
780
+ _max_concurrency_ok(
781
+ ttft_group_df, conc_col, cfg, args.ttft_max_ms, args.ttft_slack_pct
782
+ )
783
if ttft_group_df is not None
784
else pd.NA
785
)
786
tpot_max = (
631
- _max_concurrency_ok(tpot_group_df, conc_col, cfg, args.tpot_max_ms)
787
+ _max_concurrency_ok(
788
+ tpot_group_df, conc_col, cfg, args.tpot_max_ms, args.tpot_slack_pct
789
+ )
790
if tpot_group_df is not None
791
else pd.NA
792
)
@@ -657,8 +815,8 @@ def build_valid_max_concurrency_summary_df(
815
rows.append(
816
{
817
"Configuration": cfg,
660
- f"Max {conc_col} (TTFT ≤ {args.ttft_max_ms:g} ms)": ttft_max,
661
- f"Max {conc_col} (TPOT ≤ {args.tpot_max_ms:g} ms)": tpot_max,
818
+ f"Max {conc_col} (TTFT ≤ {ttft_range})": ttft_max,
819
+ f"Max {conc_col} (TPOT ≤ {tpot_range})": tpot_max,
820
f"Max {conc_col} (Both)": both,
821
"Output Tput @ Both (tok/s)": tput_at_both,
822
"TTFT @ Both (ms)": ttft_at_both,
@@ -751,7 +909,21 @@ def build_parser() -> argparse.ArgumentParser:
909
help="Reference limit for TPOT plots (ms)",
910
)
911
754
- # ---- NEW: export options ----
912
+ # ---- SLA tolerance (slack) options ----
913
+ parser.add_argument(
914
+ "--ttft-slack-pct",
915
+ type=float,
916
+ default=5.0,
917
+ help="Allowed percentage above TTFT SLA (default: 5).",
918
+ )
919
+ parser.add_argument(
920
+ "--tpot-slack-pct",
921
+ type=float,
922
+ default=5.0,
923
+ help="Allowed percentage above TPOT SLA (default: 5).",
924
+ )
925
+
926
+ # ---- export options ----
927
parser.add_argument(
928
"--excel-out",
929
type=str,
@@ -843,9 +1015,13 @@ def render_metric_table_html(
1015
1016
metric_name = metric_label.lower()
1017
if "ttft" in metric_name:
846
- styler = _highlight_threshold(display_group, args.ttft_max_ms)
1018
+ styler = _highlight_threshold(
1019
+ display_group, args.ttft_max_ms, args.ttft_slack_pct
1020
+ )
1021
elif ("tpot" in metric_name) or ("median" in metric_name) or ("p99" in metric_name):
848
- styler = _highlight_threshold(display_group, args.tpot_max_ms)
1022
+ styler = _highlight_threshold(
1023
+ display_group, args.tpot_max_ms, args.tpot_slack_pct
1024
+ )
1025
else:
1026
styler = display_group.style
1027
@@ -962,22 +1138,46 @@ def write_report_group_first(
1138
csv_dir.mkdir(parents=True, exist_ok=True)
1139
1140
excel_path = args.excel_out or "perf_comparison.xlsx"
965
- with pd.ExcelWriter(excel_path, engine="openpyxl") as xw:
1141
+ disable_excel = os.getenv("VLLM_COMPARE_DISABLE_EXCEL", "0") == "1"
1142
+
1143
+ # Prefer xlsxwriter for speed; fallback to openpyxl if unavailable.
1144
+ excel_engine = (
1145
+ os.getenv("VLLM_COMPARE_EXCEL_ENGINE", "xlsxwriter").strip() or "xlsxwriter"
1146
+ )
1147
+ if excel_engine == "xlsxwriter" and util.find_spec("xlsxwriter") is None:
1148
+ excel_engine = "openpyxl"
1149
+
1150
+ excel_engine_kwargs = {}
1151
+ if excel_engine == "xlsxwriter":
1152
+ # Reduce memory pressure & usually faster writes.
1153
+ excel_engine_kwargs = {"options": {"constant_memory": True}}
1154
+
1155
+ xw_ctx = (
1156
+ nullcontext(None)
1157
+ if disable_excel
1158
+ else pd.ExcelWriter(
1159
+ excel_path, engine=excel_engine, engine_kwargs=excel_engine_kwargs
1160
+ )
1161
+ )
1162
+ with xw_ctx as xw:
1163
+ used_sheets: set[str] = set()
1164
# ---- Environment sheet (first) ----
1165
env_sheet = _sanitize_sheet_name("Environment")
1166
env_df = _load_env_df_for_inputs(args, files)
969
- if env_df is None or env_df.empty:
970
- pd.DataFrame(
971
- [
972
- {
973
- "Section": "Environment",
974
- "Key": "vllm_env.txt",
975
- "Value": "NOT FOUND (or empty)",
976
- }
977
- ]
978
- ).to_excel(xw, sheet_name=env_sheet, index=False)
979
- else:
980
- env_df.to_excel(xw, sheet_name=env_sheet, index=False)
1167
+ if xw is not None:
1168
+ if env_df is None or env_df.empty:
1169
+ pd.DataFrame(
1170
+ [
1171
+ {
1172
+ "Section": "Environment",
1173
+ "Key": "vllm_env.txt",
1174
+ "Value": "NOT FOUND (or empty)",
1175
+ }
1176
+ ]
1177
+ ).to_excel(xw, sheet_name=env_sheet, index=False)
1178
+ else:
1179
+ env_df.to_excel(xw, sheet_name=env_sheet, index=False)
1180
+ used_sheets.add(env_sheet)
1181
with open("perf_comparison.html", "w", encoding="utf-8") as main_fh:
1182
main_fh.write('<meta charset="utf-8">\n')
1183
for gkey in group_keys:
@@ -993,12 +1193,19 @@ def write_report_group_first(
1193
1194
main_fh.write(group_header)
1195
1196
+ do_excel = xw is not None
1197
sheet = _group_to_sheet_base(group_cols_canonical, gkey_tuple)
1198
sheet_base = sheet
998
- dedup_i = 1
999
- while sheet in xw.sheets:
1000
- dedup_i += 1
1001
- sheet = _sanitize_sheet_name(f"{sheet_base}_{dedup_i}")
1199
+ if do_excel:
1200
+ dedup_i = 1
1201
+ while sheet in used_sheets:
1202
+ dedup_i += 1
1203
+ suffix = f"_{dedup_i}"
1204
+ # Ensure uniqueness even when sheet names are truncated.
1205
+ base = str(sheet_base)
1206
+ keep = max(1, 31 - len(suffix))
1207
+ sheet = _sanitize_sheet_name(base[:keep] + suffix)
1208
+ used_sheets.add(sheet)
1209
1210
excel_blocks: list[tuple[str, pd.DataFrame]] = []
1211
@@ -1059,7 +1266,7 @@ def write_report_group_first(
1266
)
1267
1268
excel_blocks.append(
1062
- (metric_label, display_group.reset_index(drop=True))
1269
+ (metric_label, group_df.reset_index(drop=True))
1270
)
1271
if csv_dir:
1272
fn = _safe_filename(
@@ -1067,7 +1274,7 @@ def write_report_group_first(
1274
"/", "_"
1275
)
1276
)
1070
- display_group.to_csv(csv_dir / f"{fn}.csv", index=False)
1277
+ group_df.to_csv(csv_dir / f"{fn}.csv", index=False)
1278
1279
summary_html = build_valid_max_concurrency_summary_html(
1280
tput_group_df=tput_group_df,
@@ -1097,9 +1304,13 @@ def write_report_group_first(
1304
)
1305
summary_df.to_csv(csv_dir / f"{fn}.csv", index=False)
1306
1100
- _write_tables_to_excel_sheet(xw, sheet, excel_blocks)
1307
+ if do_excel:
1308
+ _write_tables_to_excel_sheet(xw, sheet, excel_blocks)
1309
1102
- print(f"Wrote Excel: {excel_path}")
1310
+ if disable_excel:
1311
+ print("Skipped Excel generation (VLLM_COMPARE_DISABLE_EXCEL=1).")
1312
+ else:
1313
+ print(f"Wrote Excel: {excel_path}")
1314
if csv_dir:
1315
print(f"Wrote CSVs under: {csv_dir}")
1316
.buildkite/performance-benchmarks/scripts/run-performance-benchmarks.sh
+361
-4
@@ -12,6 +12,13 @@ DRY_RUN="${DRY_RUN:-0}"
12
MODEL_FILTER="${MODEL_FILTER:-}"
13
DTYPE_FILTER="${DTYPE_FILTER:-}"
14
15
+# Adaptive search controls
16
+ENABLE_ADAPTIVE_CONCURRENCY="${ENABLE_ADAPTIVE_CONCURRENCY:-0}"
17
+SLA_TTFT_MS="${SLA_TTFT_MS:-3000}"
18
+SLA_TPOT_MS="${SLA_TPOT_MS:-100}"
19
+ADAPTIVE_MAX_PROBES="${ADAPTIVE_MAX_PROBES:-8}"
20
+ADAPTIVE_MAX_CONCURRENCY="${ADAPTIVE_MAX_CONCURRENCY:-1024}"
21
+
22
check_gpus() {
23
if command -v nvidia-smi; then
24
# check the number of GPUs and GPU type.
@@ -183,6 +190,304 @@ upload_to_buildkite() {
190
$BUILDKITE_AGENT_COMMAND artifact upload "$RESULTS_FOLDER/*"
191
}
192
193
+# -------------------------------
194
+# Adaptive concurrency helpers
195
+# -------------------------------
196
+result_json_path_for_serving() {
197
+ local test_name=$1
198
+ local qps=$2
199
+ local max_concurrency=$3
200
+ echo "$RESULTS_FOLDER/${test_name}_qps_${qps}_concurrency_${max_concurrency}.json"
201
+}
202
+
203
+extract_metric_ms() {
204
+ local metric_name=$1
205
+ local json_file=$2
206
+
207
+ [[ -f "$json_file" ]] || return 0
208
+
209
+ if [[ "$metric_name" == "ttft" ]]; then
210
+ jq -r '
211
+ [
212
+ .ttft_ms.p99?,
213
+ .metrics.ttft_ms.p99?,
214
+ .ttft.p99?,
215
+ .metrics.ttft.p99?,
216
+ .p99_ttft_ms?,
217
+ .ttft_ms.mean?,
218
+ .metrics.ttft_ms.mean?,
219
+ .ttft.mean?,
220
+ .metrics.ttft.mean?,
221
+ .mean_ttft_ms?
222
+ ] | map(select(. != null)) | .[0] // empty
223
+ ' "$json_file"
224
+ else
225
+ jq -r '
226
+ [
227
+ .tpot_ms.p99?,
228
+ .metrics.tpot_ms.p99?,
229
+ .tpot.p99?,
230
+ .metrics.tpot.p99?,
231
+ .p99_tpot_ms?,
232
+ .itl_ms.p99?,
233
+ .metrics.itl_ms.p99?,
234
+ .inter_token_latency_ms.p99?,
235
+ .tpot_ms.mean?,
236
+ .metrics.tpot_ms.mean?,
237
+ .tpot.mean?,
238
+ .metrics.tpot.mean?,
239
+ .itl_ms.mean?,
240
+ .metrics.itl_ms.mean?,
241
+ .mean_tpot_ms?,
242
+ .mean_itl_ms?
243
+ ] | map(select(. != null)) | .[0] // empty
244
+ ' "$json_file"
245
+ fi
246
+}
247
+
248
+evaluate_sla_from_json() {
249
+ local json_file=$1
250
+ local ttft
251
+ local tpot
252
+ local pass
253
+
254
+ [[ -f "$json_file" ]] || return 2
255
+
256
+ ttft=$(extract_metric_ms ttft "$json_file")
257
+ tpot=$(extract_metric_ms tpot "$json_file")
258
+
259
+ [[ -n "$ttft" && -n "$tpot" ]] || return 2
260
+
261
+ pass=$(jq -n \
262
+ --argjson ttft "$ttft" \
263
+ --argjson tpot "$tpot" \
264
+ --argjson sla_ttft "$SLA_TTFT_MS" \
265
+ --argjson sla_tpot "$SLA_TPOT_MS" \
266
+ '($ttft <= $sla_ttft) and ($tpot <= $sla_tpot)')
267
+
268
+ [[ "$pass" == "true" ]]
269
+}
270
+
271
+write_adaptive_summary_json() {
272
+ local summary_file=$1
273
+ local test_name=$2
274
+ local qps=$3
275
+ local static_last_pass=$4
276
+ local static_first_fail=$5
277
+ local final_last_pass=$6
278
+ local final_first_fail=$7
279
+
280
+ jq -n \
281
+ --arg test_name "$test_name" \
282
+ --arg qps "$qps" \
283
+ --argjson sla_ttft "$SLA_TTFT_MS" \
284
+ --argjson sla_tpot "$SLA_TPOT_MS" \
285
+ --arg static_last_pass "${static_last_pass:-}" \
286
+ --arg static_first_fail "${static_first_fail:-}" \
287
+ --arg final_last_pass "${final_last_pass:-}" \
288
+ --arg final_first_fail "${final_first_fail:-}" \
289
+ '{
290
+ test_name: $test_name,
291
+ qps: $qps,
292
+ sla_ttft_ms: $sla_ttft,
293
+ sla_tpot_ms: $sla_tpot,
294
+ static_last_pass: (if $static_last_pass == "" then null else ($static_last_pass | tonumber) end),
295
+ static_first_fail: (if $static_first_fail == "" then null else ($static_first_fail | tonumber) end),
296
+ final_last_pass: (if $final_last_pass == "" then null else ($final_last_pass | tonumber) end),
297
+ final_first_fail: (if $final_first_fail == "" then null else ($final_first_fail | tonumber) end)
298
+ }' > "$summary_file"
299
+}
300
+
301
+run_single_serving_probe() {
302
+ local test_name=$1
303
+ local qps=$2
304
+ local max_concurrency=$3
305
+ local tp=$4
306
+ local compilation_config_mode=$5
307
+ local optimization_level=$6
308
+ local client_args_effective=$7
309
+ local client_remote_args=$8
310
+ local server_command=$9
311
+
312
+ local new_test_name="${test_name}_qps_${qps}_concurrency_${max_concurrency}"
313
+ local result_json
314
+ local num_prompts_arg=""
315
+ local client_command
316
+
317
+ result_json=$(result_json_path_for_serving "$test_name" "$qps" "$max_concurrency")
318
+
319
+ if [[ -f "$result_json" ]]; then
320
+ evaluate_sla_from_json "$result_json"
321
+ return $?
322
+ fi
323
+
324
+ if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then
325
+ num_prompts=$(( max_concurrency * PROMPTS_PER_CONCURRENCY ))
326
+ if (( num_prompts < MIN_NUM_PROMPTS )); then num_prompts=$MIN_NUM_PROMPTS; fi
327
+ if (( num_prompts > MAX_NUM_PROMPTS )); then num_prompts=$MAX_NUM_PROMPTS; fi
328
+ num_prompts_arg="--num-prompts $num_prompts"
329
+ fi
330
+
331
+ client_command="vllm bench serve \
332
+ --save-result \
333
+ --result-dir $RESULTS_FOLDER \
334
+ --result-filename ${new_test_name}.json \
335
+ --request-rate $qps \
336
+ --max-concurrency $max_concurrency \
337
+ $num_prompts_arg \
338
+ --metadata tensor_parallel_size=$tp compilation_config.mode=$compilation_config_mode optimization_level=$optimization_level adaptive_search=1 \
339
+ $client_args_effective $client_remote_args "
340
+
341
+ echo "Adaptive probe: $client_command"
342
+
343
+ if [[ "${DRY_RUN:-0}" != "1" ]]; then
344
+ bash -c "$client_command"
345
+ fi
346
+
347
+ jq_output=$(jq -n \
348
+ --arg server "$server_command" \
349
+ --arg client "$client_command" \
350
+ --arg gpu "$gpu_type" \
351
+ '{
352
+ server_command: $server,
353
+ client_command: $client,
354
+ gpu_type: $gpu,
355
+ adaptive_search: true
356
+ }')
357
+ echo "$jq_output" > "$RESULTS_FOLDER/${new_test_name}.commands"
358
+
359
+ evaluate_sla_from_json "$result_json"
360
+}
361
+
362
+adaptive_refine_from_static_results() {
363
+ local test_name=$1
364
+ local qps=$2
365
+ local max_concurrency_list_raw=$3
366
+ local tp=$4
367
+ local compilation_config_mode=$5
368
+ local optimization_level=$6
369
+ local client_args_effective=$7
370
+ local client_remote_args=$8
371
+ local server_command=$9
372
+
373
+ local sorted_points
374
+ local point
375
+ local rc
376
+ local static_last_pass=""
377
+ local static_first_fail=""
378
+ local largest_static=""
379
+ local step_hint=1
380
+ local previous_point=""
381
+ local low
382
+ local high
383
+ local mid
384
+ local probes=0
385
+ local summary_file="$RESULTS_FOLDER/${test_name}_qps_${qps}_sla_summary.json"
386
+
387
+ [[ "${ENABLE_ADAPTIVE_CONCURRENCY}" == "1" ]] || return 0
388
+ [[ "${DRY_RUN:-0}" != "1" ]] || return 0
389
+
390
+ sorted_points=$(for point in $max_concurrency_list_raw; do printf '%s\n' "$point"; done | tr -d "'" | awk '/^[0-9]+$/' | sort -n | uniq)
391
+ [[ -n "$sorted_points" ]] || return 0
392
+
393
+ while read -r point; do
394
+ [[ -z "$point" ]] && continue
395
+ largest_static="$point"
396
+ evaluate_sla_from_json "$(result_json_path_for_serving "$test_name" "$qps" "$point")"
397
+ rc=$?
398
+ if (( rc == 0 )); then
399
+ static_last_pass="$point"
400
+ elif (( rc == 1 )); then
401
+ if [[ -n "$static_last_pass" ]]; then
402
+ static_first_fail="$point"
403
+ break
404
+ fi
405
+ fi
406
+
407
+ if [[ -n "$previous_point" ]]; then
408
+ step_hint=$(( point - previous_point ))
409
+ if (( step_hint < 1 )); then step_hint=1; fi
410
+ fi
411
+ previous_point="$point"
412
+ done <<< "$sorted_points"
413
+
414
+ if [[ -z "$static_last_pass" ]]; then
415
+ write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "" "$static_first_fail" "" "$static_first_fail"
416
+ return 0
417
+ fi
418
+
419
+ if [[ -n "$static_first_fail" ]]; then
420
+ low=$static_last_pass
421
+ high=$static_first_fail
422
+ while (( low + 1 < high )) && (( probes < ADAPTIVE_MAX_PROBES )); do
423
+ mid=$(( (low + high) / 2 ))
424
+ probes=$(( probes + 1 ))
425
+ run_single_serving_probe \
426
+ "$test_name" "$qps" "$mid" "$tp" \
427
+ "$compilation_config_mode" "$optimization_level" \
428
+ "$client_args_effective" "$client_remote_args" "$server_command"
429
+ rc=$?
430
+ if (( rc == 0 )); then
431
+ low=$mid
432
+ elif (( rc == 1 )); then
433
+ high=$mid
434
+ else
435
+ break
436
+ fi
437
+ done
438
+ write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "$static_last_pass" "$static_first_fail" "$low" "$high"
439
+ return 0
440
+ fi
441
+
442
+ low=$largest_static
443
+ high=""
444
+ while (( probes < ADAPTIVE_MAX_PROBES )); do
445
+ point=$(( low + step_hint ))
446
+ if (( point > ADAPTIVE_MAX_CONCURRENCY )); then
447
+ point=$ADAPTIVE_MAX_CONCURRENCY
448
+ fi
449
+ (( point > low )) || break
450
+ probes=$(( probes + 1 ))
451
+ run_single_serving_probe \
452
+ "$test_name" "$qps" "$point" "$tp" \
453
+ "$compilation_config_mode" "$optimization_level" \
454
+ "$client_args_effective" "$client_remote_args" "$server_command"
455
+ rc=$?
456
+ if (( rc == 0 )); then
457
+ low=$point
458
+ (( point == ADAPTIVE_MAX_CONCURRENCY )) && break
459
+ step_hint=$(( step_hint * 2 ))
460
+ if (( step_hint < 1 )); then step_hint=1; fi
461
+ elif (( rc == 1 )); then
462
+ high=$point
463
+ break
464
+ else
465
+ break
466
+ fi
467
+ done
468
+
469
+ if [[ -n "$high" ]]; then
470
+ while (( low + 1 < high )) && (( probes < ADAPTIVE_MAX_PROBES )); do
471
+ mid=$(( (low + high) / 2 ))
472
+ probes=$(( probes + 1 ))
473
+ run_single_serving_probe \
474
+ "$test_name" "$qps" "$mid" "$tp" \
475
+ "$compilation_config_mode" "$optimization_level" \
476
+ "$client_args_effective" "$client_remote_args" "$server_command"
477
+ rc=$?
478
+ if (( rc == 0 )); then
479
+ low=$mid
480
+ elif (( rc == 1 )); then
481
+ high=$mid
482
+ else
483
+ break
484
+ fi
485
+ done
486
+ fi
487
+
488
+ write_adaptive_summary_json "$summary_file" "$test_name" "$qps" "$static_last_pass" "" "$low" "$high"
489
+}
490
+
491
run_benchmark_tests() {
492
# run benchmark tests using `vllm bench <test_type>` command
493
# $1: test type (latency or throughput)
@@ -347,10 +652,48 @@ run_serving_tests() {
652
server_envs=$(echo "$params" | jq -r '.server_environment_variables')
653
client_params=$(echo "$params" | jq -r '.client_parameters')
654
350
- server_args=$(json2args "$server_params")
655
+ # vLLM serve CLI: model must be positional (no --model). Convert server_parameters accordingly.
656
+ server_model=$(echo "$server_params" | jq -r '.model // empty')
657
+ if [[ -z "$server_model" || "$server_model" == "null" ]]; then
658
+ echo "Error: serving test '$test_name' is missing server_parameters.model" >&2
659
+ exit 1
660
+ fi
661
+ server_params_no_model=$(echo "$server_params" | jq -c 'del(.model)')
662
+ server_args=$(json2args "$server_params_no_model")
663
+
664
server_envs=$(json2envs "$server_envs")
665
client_args=$(json2args "$client_params")
666
667
+ # ------------------------------------------------------------
668
+ # Option 1: Dynamic num-prompts scaling based on max_concurrency
669
+ #
670
+ # If PROMPTS_PER_CONCURRENCY is set, override JSON num_prompts with:
671
+ # num_prompts = max_concurrency * PROMPTS_PER_CONCURRENCY
672
+ #
673
+ # If PROMPTS_PER_CONCURRENCY is NOT set, keep JSON num_prompts behavior
674
+ # unchanged (i.e., whatever is in serving-tests-*.json).
675
+ # ------------------------------------------------------------
676
+ PROMPTS_PER_CONCURRENCY="${PROMPTS_PER_CONCURRENCY-}" # no default on purpose
677
+ MIN_NUM_PROMPTS="${MIN_NUM_PROMPTS:-1}"
678
+ MAX_NUM_PROMPTS="${MAX_NUM_PROMPTS:-1000000}"
679
+
680
+ if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then
681
+ # Remove any fixed --num-prompts from JSON-derived args (avoid duplicates)
682
+ # Remove any fixed --num-prompts from JSON-derived args (avoid duplicates)
683
+ # Handles: --num-prompts 123 and --num-prompts=123
684
+ client_args_no_np="$(
685
+ printf ' %s ' "$client_args" \
686
+ | sed -E \
687
+ -e 's/[[:space:]]--num-prompts=([^[:space:]]+)([[:space:]]|$)/ /g' \
688
+ -e 's/[[:space:]]--num-prompts[[:space:]]+([^[:space:]]+)([[:space:]]|$)/ /g'
689
+ )"
690
+ # normalize whitespace
691
+ client_args_no_np="$(echo "$client_args_no_np" | tr -s ' ' | sed -E 's/^ //; s/ $//')"
692
+ client_args_no_np="$(echo "$client_args_no_np" | xargs)"
693
+ client_args_effective="$client_args_no_np"
694
+ else
695
+ client_args_effective="$client_args"
696
+ fi
697
# qps_list
698
qps_list=$(echo "$params" | jq -r '.qps_list')
699
qps_list=$(echo "$qps_list" | jq -r '.[] | @sh')
@@ -382,14 +725,13 @@ run_serving_tests() {
725
fi
726
727
# check if server model and client model is aligned
385
- server_model=$(echo "$server_params" | jq -r '.model')
728
client_model=$(echo "$client_params" | jq -r '.model')
729
if [[ $server_model != "$client_model" ]]; then
730
echo "Server model and client model must be the same. Skip testcase $test_name."
731
continue
732
fi
733
392
- server_command="$server_envs vllm serve \
734
+ server_command="$server_envs vllm serve $server_model \
735
$server_args"
736
737
# run the server
@@ -436,6 +778,14 @@ run_serving_tests() {
778
for max_concurrency in $max_concurrency_list; do
779
new_test_name="${test_name}_qps_${qps}_concurrency_${max_concurrency}"
780
echo " new test name $new_test_name"
781
+ # If PROMPTS_PER_CONCURRENCY is set, compute per-concurrency --num-prompts.
782
+ num_prompts_arg=""
783
+ if [[ -n "${PROMPTS_PER_CONCURRENCY}" ]]; then
784
+ num_prompts=$(( max_concurrency * PROMPTS_PER_CONCURRENCY ))
785
+ if (( num_prompts < MIN_NUM_PROMPTS )); then num_prompts=$MIN_NUM_PROMPTS; fi
786
+ if (( num_prompts > MAX_NUM_PROMPTS )); then num_prompts=$MAX_NUM_PROMPTS; fi
787
+ num_prompts_arg="--num-prompts $num_prompts"
788
+ fi
789
# pass the tensor parallel size, the compilation mode, and the optimization
790
# level to the client so that they can be used on the benchmark dashboard
791
client_command="vllm bench serve \
@@ -444,8 +794,9 @@ run_serving_tests() {
794
--result-filename ${new_test_name}.json \
795
--request-rate $qps \
796
--max-concurrency $max_concurrency \
797
+ $num_prompts_arg \
798
--metadata tensor_parallel_size=$tp compilation_config.mode=$compilation_config_mode optimization_level=$optimization_level \
448
- $client_args $client_remote_args "
799
+ $client_args_effective $client_remote_args "
800
801
echo "Running test case $test_name with qps $qps"
802
echo "Client command: $client_command"
@@ -467,6 +818,11 @@ run_serving_tests() {
818
echo "$jq_output" >"$RESULTS_FOLDER/${new_test_name}.commands"
819
820
done
821
+
822
+ adaptive_refine_from_static_results \
823
+ "$test_name" "$qps" "$max_concurrency_list" "$tp" \
824
+ "$compilation_config_mode" "$optimization_level" \
825
+ "$client_args_effective" "$client_remote_args" "$server_command"
826
done
827
828
# clean up
@@ -532,6 +888,7 @@ main() {
888
# postprocess benchmarking results
889
pip install tabulate pandas
890
python3 $QUICK_BENCHMARK_ROOT/scripts/convert-results-json-to-markdown.py
891
+ python3 $QUICK_BENCHMARK_ROOT/scripts/compare-json-results.py -f $RESULTS_FOLDER/benchmark_results.json
892
893
upload_to_buildkite
894
}
.buildkite/performance-benchmarks/tests/serving-tests-cpu-asr.json
new
+37
@@ -0,0 +1,37 @@
1
+{
2
+ "defaults": {
3
+ "qps_list": [
4
+ "inf"
5
+ ],
6
+ "max_concurrency_list": [12, 16, 24, 32, 64, 128, 200],
7
+ "server_environment_variables": {
8
+ "VLLM_RPC_TIMEOUT": 100000,
9
+ "VLLM_ENGINE_ITERATION_TIMEOUT_S": 120
10
+ },
11
+ "server_parameters": {
12
+ "dtype": "bfloat16",
13
+ "model": "openai/whisper-large-v3-turbo"
14
+ },
15
+ "client_parameters": {
16
+ "model": "openai/whisper-large-v3-turbo",
17
+ "backend": "openai-audio",
18
+ "endpoint": "/v1/audio/transcriptions",
19
+ "dataset_name": "hf",
20
+ "dataset_path": "openslr/librispeech_asr",
21
+ "hf_subset": "clean",
22
+ "hf_split": "test",
23
+ "no_stream": "",
24
+ "no_oversample": "",
25
+ "num_prompts": 200
26
+ }
27
+ },
28
+ "tests": [
29
+ {
30
+ "test_name": "serving_whisper_large_v3_turbo_librispeech_clean_tp1",
31
+ "server_parameters": {
32
+ "tensor_parallel_size": 1
33
+ },
34
+ "client_parameters": {}
35
+ }
36
+ ]
37
+}
.buildkite/performance-benchmarks/tests/serving-tests-cpu-text.json
+72
@@ -149,6 +149,39 @@
149
"random-output-len": 128
150
}
151
},
152
+ {
153
+ "test_name": "serving_llama8B_tp1_random_2048_2048",
154
+ "server_parameters": {
155
+ "tensor_parallel_size": 1
156
+ },
157
+ "client_parameters": {
158
+ "dataset_name": "random",
159
+ "random-input-len": 2048,
160
+ "random-output-len": 2048
161
+ }
162
+ },
163
+ {
164
+ "test_name": "serving_llama8B_tp2_random_2048_2048",
165
+ "server_parameters": {
166
+ "tensor_parallel_size": 2
167
+ },
168
+ "client_parameters": {
169
+ "dataset_name": "random",
170
+ "random-input-len": 2048,
171
+ "random-output-len": 2048
172
+ }
173
+ },
174
+ {
175
+ "test_name": "serving_llama8B_tp4_random_2048_2048",
176
+ "server_parameters": {
177
+ "tensor_parallel_size": 4
178
+ },
179
+ "client_parameters": {
180
+ "dataset_name": "random",
181
+ "random-input-len": 2048,
182
+ "random-output-len": 2048
183
+ }
184
+ },
185
{
186
"test_name": "serving_llama8B_int4_tp1_random_128_128",
187
"server_parameters": {
@@ -188,6 +221,45 @@
221
"random-output-len": 128
222
}
223
},
224
+ {
225
+ "test_name": "serving_llama8B_int8_tp1_random_128_128",
226
+ "server_parameters": {
227
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
228
+ "tensor_parallel_size": 1
229
+ },
230
+ "client_parameters": {
231
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
232
+ "dataset_name": "random",
233
+ "random-input-len": 128,
234
+ "random-output-len": 128
235
+ }
236
+ },
237
+ {
238
+ "test_name": "serving_llama8B_int8_tp2_random_128_128",
239
+ "server_parameters": {
240
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
241
+ "tensor_parallel_size": 2
242
+ },
243
+ "client_parameters": {
244
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
245
+ "dataset_name": "random",
246
+ "random-input-len": 128,
247
+ "random-output-len": 128
248
+ }
249
+ },
250
+ {
251
+ "test_name": "serving_llama8B_int8_tp4_random_128_128",
252
+ "server_parameters": {
253
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
254
+ "tensor_parallel_size": 4
255
+ },
256
+ "client_parameters": {
257
+ "model": "RedHatAI/Meta-Llama-3.1-8B-Instruct-quantized.w8a8",
258
+ "dataset_name": "random",
259
+ "random-input-len": 128,
260
+ "random-output-len": 128
261
+ }
262
+ },
263
{
264
"test_name": "serving_llama3B_tp1_random_128_128",
265
"server_parameters": {
.buildkite/performance-benchmarks/tests/serving-tests-cpu.json
+12
-23
@@ -72,17 +72,6 @@
72
"random-output-len": 128
73
}
74
},
75
- {
76
- "test_name": "serving_llama8B_tp4_random_128_128",
77
- "server_parameters": {
78
- "tensor_parallel_size": 4
79
- },
80
- "client_parameters": {
81
- "dataset_name": "random",
82
- "random-input-len": 128,
83
- "random-output-len": 128
84
- }
85
- },
75
{
76
"test_name": "serving_llama8B_tp1_random_128_2048",
77
"server_parameters": {
@@ -106,20 +95,20 @@
95
}
96
},
97
{
109
- "test_name": "serving_llama8B_tp4_random_128_2048",
98
+ "test_name": "serving_llama8B_tp1_random_2048_128",
99
"server_parameters": {
111
- "tensor_parallel_size": 4
100
+ "tensor_parallel_size": 1
101
},
102
"client_parameters": {
103
"dataset_name": "random",
115
- "random-input-len": 128,
116
- "random-output-len": 2048
104
+ "random-input-len": 2048,
105
+ "random-output-len": 128
106
}
107
},
108
{
120
- "test_name": "serving_llama8B_tp1_random_2048_128",
109
+ "test_name": "serving_llama8B_tp2_random_2048_128",
110
"server_parameters": {
122
- "tensor_parallel_size": 1
111
+ "tensor_parallel_size": 2
112
},
113
"client_parameters": {
114
"dataset_name": "random",
@@ -128,25 +117,25 @@
117
}
118
},
119
{
131
- "test_name": "serving_llama8B_tp2_random_2048_128",
120
+ "test_name": "serving_llama8B_tp1_random_2048_2048",
121
"server_parameters": {
133
- "tensor_parallel_size": 2
122
+ "tensor_parallel_size": 1
123
},
124
"client_parameters": {
125
"dataset_name": "random",
126
"random-input-len": 2048,
138
- "random-output-len": 128
127
+ "random-output-len": 2048
128
}
129
},
130
{
142
- "test_name": "serving_llama8B_tp4_random_2048_128",
131
+ "test_name": "serving_llama8B_tp2_random_2048_2048",
132
"server_parameters": {
144
- "tensor_parallel_size": 4
133
+ "tensor_parallel_size": 2
134
},
135
"client_parameters": {
136
"dataset_name": "random",
137
"random-input-len": 2048,
149
- "random-output-len": 128
138
+ "random-output-len": 2048
139
}
140
}
141
]
docs/benchmarking/dashboard.md
+6
@@ -39,6 +39,12 @@ When run, benchmark script generates results under **benchmark/results** folder,
39
- `THROUGHPUT_JSON`: JSON file to use for the throughout tests. Default value is empty string (use default file).
40
- `REMOTE_HOST`: IP for the remote vLLM service to benchmark. Default value is empty string.
41
- `REMOTE_PORT`: Port for the remote vLLM service to benchmark. Default value is empty string.
42
+- `PROMPTS_PER_CONCURRENCY`: Multiplier to compute `num_prompts` for serving tests (`num_prompts = max_concurrency × value`). Overrides JSON `num_prompts`. Default is NULL.
43
+- `ENABLE_ADAPTIVE_CONCURRENCY`: set the value to '1' to enable adaptive SLA-based concurrency search after the static serving max_concurrency sweep. Default value is 0.
44
+- `SLA_TTFT_MS`: default TTFT SLA threshold in milliseconds for adaptive concurrency search. Default value is 3000.
45
+- `SLA_TPOT_MS`: default TPOT SLA threshold in milliseconds for adaptive concurrency search. Default value is 100.
46
+- `ADAPTIVE_MAX_PROBES`: maximum number of extra adaptive search probes. Default value is 8.
47
+- `ADAPTIVE_MAX_CONCURRENCY`: maximum allowed concurrency during adaptive search. Default value is 1024.
48
49
### Visualization
50
requirements/test.in
+4
-1
@@ -70,4 +70,7 @@ kaldi-native-fbank >= 1.18.7 # required for fireredasr2 test
70
71
# Newer versions of datasets require torchcoded, that makes the tests fail in CI because of a missing library.
72
# Older versions are in conflict with teerratorch requirements.
73
-datasets>=3.3.0,<=3.6.0
\ No newline at end of file
73
+datasets>=3.3.0,<=3.6.0
74
+
75
+openpyxl # required for perf comparison excel report
76
+plotly # required for perf comparison html report
requirements/test.txt
+7
-1
@@ -202,6 +202,8 @@ email-validator==2.2.0
202
# via pydantic
203
encodec==0.1.1
204
# via vocos
205
+et-xmlfile==2.0.0
206
+ # via openpyxl
207
evaluate==0.4.3
208
# via lm-eval
209
fastapi==0.128.0
@@ -634,6 +636,8 @@ opencv-python-headless==4.13.0.90
636
# albucore
637
# albumentations
638
# mistral-common
639
+openpyxl==3.1.5
640
+ # via -r requirements/test.in
641
opentelemetry-api==1.35.0
642
# via
643
# opentelemetry-exporter-prometheus
@@ -734,7 +738,9 @@ platformdirs==4.3.6
738
# virtualenv
739
# wandb
740
plotly==5.24.1
737
- # via genai-perf
741
+ # via
742
+ # -r requirements/test.in
743
+ # genai-perf
744
pluggy==1.5.0
745
# via
746
# pytest