@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 02acd16861
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 7, 16:30 UTC

[Benchmarks] Plot benchmark timeline and requests statistics (#35220)

Signed-off-by: Sophie du Couédic <sop@zurich.ibm.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Sophie du Couédic committed Feb 26, 2026 at 11:17 UTC 02acd16861bc6388ab79b6d7c9abb20c0237426e
3 files changed +466 -18
setup.py
+1 -1
@@ -1033,7 +1033,7 @@ setup(
1033 ext_modules=ext_modules,
1034 install_requires=get_requirements(),
1035 extras_require={
1036 - "bench": ["pandas", "matplotlib", "seaborn", "datasets", "scipy"],
1036 + "bench": ["pandas", "matplotlib", "seaborn", "datasets", "scipy", "plotly"],
1037 "tensorizer": ["tensorizer==2.10.1"],
1038 "fastsafetensors": ["fastsafetensors >= 0.2.2"],
1039 "runai": ["runai-model-streamer[s3,gcs] >= 0.15.3"],
vllm/benchmarks/plot.py new
+316
@@ -0,0 +1,316 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Generate plots for benchmark results."""
4 +
5 +from pathlib import Path
6 +from typing import Any
7 +
8 +from vllm.utils.import_utils import PlaceholderModule
9 +
10 +try:
11 + import plotly.express as px
12 + import plotly.io as pio
13 +except ImportError:
14 + _plotly = PlaceholderModule("plotly")
15 + px = _plotly.placeholder_attr("express")
16 + pio = _plotly.placeholder_attr("io")
17 +
18 +try:
19 + import matplotlib.pyplot as plt
20 +except ImportError:
21 + _matplotlib = PlaceholderModule("matplotlib")
22 + plt = _matplotlib.placeholder_attr("pyplot")
23 +
24 +
25 +def generate_timeline_plot(
26 + results: list[dict[str, Any]],
27 + output_path: Path,
28 + colors: list[str] | None = None,
29 + itl_thresholds: list[float] | None = None,
30 + labels: list[str] | None = None,
31 +) -> None:
32 + """
33 + Generate an HTML timeline plot from benchmark results.
34 +
35 + Args:
36 + results: List of per-request result dictionaries containing:
37 + - start_time: Request start time (seconds)
38 + - ttft: Time to first token (seconds)
39 + - itl: List of inter-token latencies (seconds)
40 + - latency: Total request latency (seconds)
41 + - prompt_len: Number of prompt tokens
42 + - output_tokens: Number of output tokens
43 + output_path: Path where the HTML file will be saved
44 + colors: List of colors for ITL categories (default: green, orange, red, black)
45 + itl_thresholds: ITL thresholds in seconds (default: [1.0, 4.0, 6.0])
46 + labels: Labels for ITL categories (default based on thresholds)
47 + """
48 +
49 + # Set defaults
50 + if colors is None:
51 + colors = ["#109618", "#FF7F0E", "#D62728"]
52 + if itl_thresholds is None:
53 + itl_thresholds = [0.025, 0.050]
54 + if labels is None:
55 + labels = [
56 + f"ITL < {itl_thresholds[0] * 1000:.0f}ms",
57 + f"{itl_thresholds[0] * 1000:.0f}ms ≤ ITL < {itl_thresholds[1] * 1000:.0f}ms", # noqa
58 + f"ITL ≥ {itl_thresholds[1] * 1000:.0f}ms",
59 + ]
60 +
61 + labels_colors = {"TTFT": "#636EFA", **dict(zip(labels, colors))}
62 + labels_order = ["TTFT"] + labels
63 +
64 + timeline_data = construct_timeline_data(results, itl_thresholds, labels)
65 +
66 + if not timeline_data:
67 + print("No timeline data to plot")
68 + return
69 +
70 + # Create the plot
71 + fig = px.timeline(
72 + timeline_data,
73 + x_start="start",
74 + x_end="end",
75 + y="request_id",
76 + color="type",
77 + color_discrete_map=labels_colors,
78 + category_orders={"type": labels_order},
79 + hover_data=[
80 + "prompt_tokens",
81 + "output_tokens",
82 + "req_start_time",
83 + "req_finish_time",
84 + "segment_start",
85 + "segment_end",
86 + "duration",
87 + ],
88 + )
89 +
90 + # Customize hover template to show only time without date
91 + fig.update_traces(
92 + hovertemplate="<b>%{y}</b><br>"
93 + "Type: %{fullData.name}<br>"
94 + "Start: %{customdata[4]}<br>"
95 + "End: %{customdata[5]}<br>"
96 + "Duration: %{customdata[6]}<br>"
97 + "Prompt Tokens: %{customdata[0]}<br>"
98 + "Output Tokens: %{customdata[1]}<br>"
99 + "Request Start Time: %{customdata[2]}<br>"
100 + "Request End Time: %{customdata[3]}<br>"
101 + "<extra></extra>"
102 + )
103 +
104 + fig.update_yaxes(autorange="reversed")
105 + fig.update_layout(
106 + xaxis_title="Time",
107 + yaxis_title="Request ID",
108 + showlegend=True,
109 + )
110 +
111 + # Save to HTML
112 + pio.write_html(fig, str(output_path))
113 + print(f"Timeline plot saved to: {output_path}")
114 +
115 +
116 +def construct_timeline_data(
117 + requests_data: list[dict[str, Any]],
118 + itl_thresholds: list[float],
119 + labels: list[str],
120 +) -> list[dict[str, Any]]:
121 + """
122 + Construct timeline data from request results.
123 +
124 + Args:
125 + requests_data: List of per-request result dictionaries
126 + itl_thresholds: ITL thresholds in seconds
127 + labels: Labels for ITL categories
128 +
129 + Returns:
130 + List of timeline segments for plotting
131 + """
132 +
133 + def tostr(sec_time: float) -> str:
134 + """Convert seconds to HH:MM:SS.mmm format."""
135 + h = int(sec_time // 3600)
136 + assert h < 100, "time seems to last more than 100 hours"
137 + m = int((sec_time % 3600) // 60)
138 + s = sec_time % 60
139 + return f"{h:02d}:{m:02d}:{s:06.3f}"
140 +
141 + def itl_type(itl: float) -> str:
142 + """Categorize ITL based on thresholds."""
143 + if itl < itl_thresholds[0]:
144 + return labels[0]
145 + elif itl < itl_thresholds[1]:
146 + return labels[1]
147 + else:
148 + return labels[2]
149 +
150 + # Find the earliest start time to use as t0
151 + t0 = None
152 + for request in requests_data:
153 + start_time = request.get("start_time")
154 + if start_time is not None and (t0 is None or start_time < t0):
155 + t0 = start_time
156 +
157 + if t0 is None:
158 + return []
159 +
160 + timeline_data = []
161 +
162 + for i, request in enumerate(requests_data):
163 + start_time = request.get("start_time")
164 + ttft = request.get("ttft")
165 + itl = request.get("itl", [])
166 + latency = request.get("latency")
167 + prompt_len = request.get("prompt_len", 0)
168 + output_tokens = request.get("output_tokens", 0)
169 +
170 + # Skip requests without required data
171 + if start_time is None or ttft is None or latency is None:
172 + continue
173 +
174 + # Normalize start time
175 + start_time = start_time - t0
176 + start_time_str = tostr(start_time)
177 +
178 + # TTFT segment
179 + ttft_end = start_time + ttft
180 + ttft_end_str = tostr(ttft_end)
181 +
182 + timeline_data.append(
183 + {
184 + "request_id": f"Req {i}",
185 + "start": start_time_str,
186 + "end": ttft_end_str,
187 + "type": "TTFT",
188 + "prompt_tokens": prompt_len,
189 + "output_tokens": output_tokens,
190 + "req_start_time": tostr(start_time),
191 + "req_finish_time": tostr(start_time + latency),
192 + "segment_start": start_time_str,
193 + "segment_end": ttft_end_str,
194 + "duration": f"{ttft:.3f}s",
195 + }
196 + )
197 +
198 + # ITL segments
199 + prev_time = ttft_end
200 + prev_time_str = ttft_end_str
201 +
202 + for itl_value in itl:
203 + itl_end = prev_time + itl_value
204 + itl_end_str = tostr(itl_end)
205 +
206 + timeline_data.append(
207 + {
208 + "request_id": f"Req {i}",
209 + "start": prev_time_str,
210 + "end": itl_end_str,
211 + "type": itl_type(itl_value),
212 + "prompt_tokens": prompt_len,
213 + "output_tokens": output_tokens,
214 + "req_start_time": tostr(start_time),
215 + "req_finish_time": tostr(start_time + latency),
216 + "segment_start": prev_time_str,
217 + "segment_end": itl_end_str,
218 + "duration": f"{itl_value:.3f}s",
219 + }
220 + )
221 +
222 + prev_time = itl_end
223 + prev_time_str = itl_end_str
224 +
225 + return timeline_data
226 +
227 +
228 +def generate_dataset_stats_plot(
229 + results: list[dict[str, Any]],
230 + output_path: Path,
231 +) -> None:
232 + """
233 + Generate a matplotlib figure with dataset statistics.
234 +
235 + Creates a figure with 4 subplots:
236 + - Top-left: Prompt tokens distribution (histogram)
237 + - Top-right: Output tokens distribution (histogram)
238 + - Bottom-left: Prompt+output tokens distribution (histogram)
239 + - Bottom-right: Stacked bar chart (request_id vs tokens)
240 +
241 + Args:
242 + results: List of per-request result dictionaries containing:
243 + - prompt_len: Number of prompt tokens
244 + - output_tokens: Number of output tokens
245 + output_path: Path where the figure will be saved
246 + """
247 + # Extract data
248 + prompt_tokens = []
249 + output_tokens = []
250 + total_tokens = []
251 +
252 + for request in results:
253 + prompt_len = request.get("prompt_len", 0)
254 + output_len = request.get("output_tokens", 0)
255 +
256 + prompt_tokens.append(prompt_len)
257 + output_tokens.append(output_len)
258 + total_tokens.append(prompt_len + output_len)
259 +
260 + if not prompt_tokens:
261 + print("No data available for dataset statistics plot")
262 + return
263 +
264 + # Create figure with 4 subplots
265 + fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(14, 10))
266 +
267 + # Top-left: Prompt tokens distribution
268 + ax1.hist(prompt_tokens, bins=30, color="steelblue", edgecolor="black", alpha=0.7)
269 + ax1.set_xlabel("Prompt Tokens")
270 + ax1.set_ylabel("Frequency")
271 + ax1.set_title("Prompt Tokens Distribution")
272 + ax1.grid(True, alpha=0.3)
273 +
274 + # Top-right: Output tokens distribution
275 + ax2.hist(output_tokens, bins=30, color="coral", edgecolor="black", alpha=0.7)
276 + ax2.set_xlabel("Output Tokens")
277 + ax2.set_ylabel("Frequency")
278 + ax2.set_title("Output Tokens Distribution")
279 + ax2.grid(True, alpha=0.3)
280 +
281 + # Bottom-left: Prompt+output tokens distribution
282 + ax3.hist(
283 + total_tokens, bins=30, color="mediumseagreen", edgecolor="black", alpha=0.7
284 + )
285 + ax3.set_xlabel("Total Tokens (Prompt + Output)")
286 + ax3.set_ylabel("Frequency")
287 + ax3.set_title("Total Tokens Distribution")
288 + ax3.grid(True, alpha=0.3)
289 +
290 + # Bottom-right: Stacked bar chart
291 + request_ids = list(range(len(prompt_tokens)))
292 + ax4.bar(
293 + request_ids, prompt_tokens, label="Prompt Tokens", color="steelblue", alpha=0.7
294 + )
295 + ax4.bar(
296 + request_ids,
297 + output_tokens,
298 + bottom=prompt_tokens,
299 + label="Output Tokens",
300 + color="coral",
301 + alpha=0.7,
302 + )
303 + ax4.set_xlabel("Request ID")
304 + ax4.set_ylabel("Tokens")
305 + ax4.set_title("Tokens per Request (Stacked)")
306 + ax4.legend()
307 + ax4.grid(True, alpha=0.3, axis="y")
308 +
309 + # Adjust layout to prevent overlap
310 + plt.tight_layout()
311 +
312 + # Save figure
313 + plt.savefig(str(output_path), dpi=150, bbox_inches="tight")
314 + plt.close(fig)
315 +
316 + print(f"Dataset statistics plot saved to: {output_path}")
vllm/benchmarks/serve.py
+149 -17
@@ -34,6 +34,7 @@ from collections.abc import AsyncGenerator, Iterable
34 from dataclasses import dataclass
35 from datetime import datetime
36 from enum import Enum
37 +from pathlib import Path
38 from typing import Any, Literal
39
40 import aiohttp
@@ -1183,6 +1184,49 @@ def save_to_pytorch_benchmark_format(
1184 write_to_json(pt_file, pt_records)
1185
1186
1187 +def compute_result_filename(
1188 + args: argparse.Namespace,
1189 + model_id: str,
1190 + label: str,
1191 + current_dt: str,
1192 +) -> str | None:
1193 + """Compute the result filename based on benchmark configuration.
1194 +
1195 + Args:
1196 + args: Command line arguments containing result configuration
1197 + model_id: The model identifier
1198 + label: The benchmark label
1199 + current_dt: Current datetime string
1200 +
1201 + Returns:
1202 + The computed filename path or None if no result saving is requested
1203 + """
1204 + if not (args.plot_timeline or args.save_result or args.append_result):
1205 + return None
1206 +
1207 + base_model_id = model_id.split("/")[-1]
1208 + max_concurrency_str = (
1209 + f"-concurrency{args.max_concurrency}"
1210 + if args.max_concurrency is not None
1211 + else ""
1212 + )
1213 + label = label or args.backend
1214 +
1215 + if args.ramp_up_strategy is not None:
1216 + file_name = f"{label}-ramp-up-{args.ramp_up_strategy}-{args.ramp_up_start_rps}qps-{args.ramp_up_end_rps}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa
1217 + else:
1218 + file_name = f"{label}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa
1219 +
1220 + if args.result_filename:
1221 + file_name = args.result_filename
1222 +
1223 + if args.result_dir:
1224 + os.makedirs(args.result_dir, exist_ok=True)
1225 + file_name = os.path.join(args.result_dir, file_name)
1226 +
1227 + return file_name
1228 +
1229 +
1230 def add_cli_args(parser: argparse.ArgumentParser):
1231 add_dataset_parser(parser)
1232 parser.add_argument(
@@ -1535,6 +1579,30 @@ def add_cli_args(parser: argparse.ArgumentParser):
1579 "connecting to servers with self-signed certificates.",
1580 )
1581
1582 + parser.add_argument(
1583 + "--plot-timeline",
1584 + action="store_true",
1585 + help="Generate an HTML timeline plot showing request execution. "
1586 + "The plot will be saved alongside the results JSON file.",
1587 + )
1588 + parser.add_argument(
1589 + "--timeline-itl-thresholds",
1590 + type=float,
1591 + nargs=2,
1592 + default=[25.0, 50.0],
1593 + metavar=("THRESHOLD1", "THRESHOLD2"),
1594 + help="ITL thresholds in milliseconds for timeline plot coloring. "
1595 + "Specify two values to categorize inter-token latencies into three groups: "
1596 + "below first threshold (green), between thresholds (orange), "
1597 + "and above second threshold (red). Default: 25 50 (milliseconds).",
1598 + )
1599 + parser.add_argument(
1600 + "--plot-dataset-stats",
1601 + action="store_true",
1602 + help="Generate a matplotlib figure with dataset statistics showing "
1603 + "prompt tokens, output tokens, and combined token distributions.",
1604 + )
1605 +
1606
1607 def main(args: argparse.Namespace) -> dict[str, Any]:
1608 return asyncio.run(main_async(args))
@@ -1770,6 +1838,86 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1838 # Merge with benchmark result
1839 result_json = {**result_json, **benchmark_result}
1840
1841 + # Compute file_name once before using it for plots or saving results
1842 + file_name = compute_result_filename(args, model_id, label, current_dt)
1843 +
1844 + # Generate timeline plot if requested
1845 + if args.plot_timeline:
1846 + try:
1847 + from vllm.benchmarks.plot import generate_timeline_plot
1848 +
1849 + # Prepare per-request data for timeline
1850 + per_request_data = []
1851 + start_times = benchmark_result.get("start_times", [])
1852 + ttfts = benchmark_result.get("ttfts", [])
1853 + itls = benchmark_result.get("itls", [])
1854 + input_lens = benchmark_result.get("input_lens", [])
1855 + output_lens = benchmark_result.get("output_lens", [])
1856 +
1857 + if start_times and ttfts and itls:
1858 + for i in range(len(start_times)):
1859 + # Calculate latency as ttft + sum of all itls
1860 + latency = ttfts[i] + sum(itls[i]) if itls[i] else ttfts[i]
1861 +
1862 + per_request_data.append(
1863 + {
1864 + "start_time": start_times[i],
1865 + "ttft": ttfts[i],
1866 + "itl": itls[i],
1867 + "latency": latency,
1868 + "prompt_len": input_lens[i],
1869 + "output_tokens": output_lens[i],
1870 + }
1871 + )
1872 +
1873 + timeline_path = Path(file_name).with_suffix(".timeline.html")
1874 + # Convert thresholds from milliseconds to seconds
1875 + itl_thresholds_sec = [t / 1000.0 for t in args.timeline_itl_thresholds]
1876 + generate_timeline_plot(
1877 + per_request_data, timeline_path, itl_thresholds=itl_thresholds_sec
1878 + )
1879 + else:
1880 + warnings.warn(
1881 + "Timeline plot requires detailed metrics. "
1882 + "Ensure the benchmark completed successfully.",
1883 + stacklevel=2,
1884 + )
1885 + except Exception as e:
1886 + warnings.warn(f"Failed to generate timeline plot: {e}", stacklevel=2)
1887 +
1888 + # Generate dataset statistics plot if requested
1889 + if args.plot_dataset_stats:
1890 + try:
1891 + from vllm.benchmarks.plot import generate_dataset_stats_plot
1892 +
1893 + # Prepare per-request data for dataset stats
1894 + per_request_data = []
1895 + input_lens = benchmark_result.get("input_lens", [])
1896 + output_lens = benchmark_result.get("output_lens", [])
1897 +
1898 + if input_lens and output_lens:
1899 + for req_input_len, req_output_len in zip(input_lens, output_lens):
1900 + per_request_data.append(
1901 + {
1902 + "prompt_len": req_input_len,
1903 + "output_tokens": req_output_len,
1904 + }
1905 + )
1906 +
1907 + stats_path = Path(file_name).with_suffix(".dataset_stats.png")
1908 + generate_dataset_stats_plot(per_request_data, stats_path)
1909 + else:
1910 + warnings.warn(
1911 + "Dataset statistics plot requires input and "
1912 + "output length data. Ensure the benchmark completed "
1913 + "successfully.",
1914 + stacklevel=2,
1915 + )
1916 + except Exception as e:
1917 + warnings.warn(
1918 + f"Failed to generate dataset statistics plot: {e}", stacklevel=2
1919 + )
1920 +
1921 if not args.save_detailed:
1922 # Remove fields with too many data points
1923 for field in [
@@ -1786,24 +1934,8 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1934 if field in benchmark_result:
1935 del benchmark_result[field]
1936
1789 - # Save to file
1937 + # Save to file
1938 if args.save_result or args.append_result:
1791 - base_model_id = model_id.split("/")[-1]
1792 - max_concurrency_str = (
1793 - f"-concurrency{args.max_concurrency}"
1794 - if args.max_concurrency is not None
1795 - else ""
1796 - )
1797 - label = label or args.backend
1798 - if args.ramp_up_strategy is not None:
1799 - file_name = f"{label}-ramp-up-{args.ramp_up_strategy}-{args.ramp_up_start_rps}qps-{args.ramp_up_end_rps}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa
1800 - else:
1801 - file_name = f"{label}-{args.request_rate}qps{max_concurrency_str}-{base_model_id}-{current_dt}.json" # noqa
1802 - if args.result_filename:
1803 - file_name = args.result_filename
1804 - if args.result_dir:
1805 - os.makedirs(args.result_dir, exist_ok=True)
1806 - file_name = os.path.join(args.result_dir, file_name)
1939 with open(
1940 file_name, mode="a+" if args.append_result else "w", encoding="utf-8"
1941 ) as outfile: