| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Generate plots for benchmark results.""" |
| 4 | |
| 5 | from pathlib import Path |
| 6 | from typing import Any |
| 7 | |
| 8 | from vllm.utils.import_utils import PlaceholderModule |
| 9 | |
| 10 | try: |
| 11 | import plotly.express as px |
| 12 | import plotly.io as pio |
| 13 | except ImportError: |
| 14 | _plotly = PlaceholderModule("plotly") |
| 15 | px = _plotly.placeholder_attr("express") |
| 16 | pio = _plotly.placeholder_attr("io") |
| 17 | |
| 18 | try: |
| 19 | import matplotlib.pyplot as plt |
| 20 | except ImportError: |
| 21 | _matplotlib = PlaceholderModule("matplotlib") |
| 22 | plt = _matplotlib.placeholder_attr("pyplot") |
| 23 | |
| 24 | |
| 25 | def generate_timeline_plot( |
| 26 | results: list[dict[str, Any]], |
| 27 | output_path: Path, |
| 28 | colors: list[str] | None = None, |
| 29 | itl_thresholds: list[float] | None = None, |
| 30 | labels: list[str] | None = None, |
| 31 | ) -> None: |
| 32 | """Generate an HTML timeline plot from benchmark results. |
| 33 | |
| 34 | Args: |
| 35 | results: List of per-request result dictionaries containing: |
| 36 | - start_time: Request start time (seconds) |
| 37 | - ttft: Time to first token (seconds) |
| 38 | - itl: List of inter-token latencies (seconds) |
| 39 | - latency: Total request latency (seconds) |
| 40 | - prompt_len: Number of prompt tokens |
| 41 | - output_tokens: Number of output tokens |
| 42 | output_path: Path where the HTML file will be saved |
| 43 | colors: List of colors for ITL categories (default: green, orange, red, black) |
| 44 | itl_thresholds: ITL thresholds in seconds (default: [1.0, 4.0, 6.0]) |
| 45 | labels: Labels for ITL categories (default based on thresholds) |
| 46 | |
| 47 | """ |
| 48 | # Set defaults |
| 49 | if colors is None: |
| 50 | colors = ["#109618", "#FF7F0E", "#D62728"] |
| 51 | if itl_thresholds is None: |
| 52 | itl_thresholds = [0.025, 0.050] |
| 53 | if labels is None: |
| 54 | labels = [ |
| 55 | f"ITL < {itl_thresholds[0] * 1000:.0f}ms", |
| 56 | f"{itl_thresholds[0] * 1000:.0f}ms ≤ ITL < {itl_thresholds[1] * 1000:.0f}ms", # noqa |
| 57 | f"ITL ≥ {itl_thresholds[1] * 1000:.0f}ms", |
| 58 | ] |
| 59 | |
| 60 | labels_colors = {"TTFT": "#636EFA", **dict(zip(labels, colors))} |
| 61 | labels_order = ["TTFT"] + labels |
| 62 | |
| 63 | timeline_data = construct_timeline_data(results, itl_thresholds, labels) |
| 64 | |
| 65 | if not timeline_data: |
| 66 | print("No timeline data to plot") |
| 67 | return |
| 68 | |
| 69 | # Create the plot |
| 70 | fig = px.timeline( |
| 71 | timeline_data, |
| 72 | x_start="start", |
| 73 | x_end="end", |
| 74 | y="request_id", |
| 75 | color="type", |
| 76 | color_discrete_map=labels_colors, |
| 77 | category_orders={"type": labels_order}, |
| 78 | hover_data=[ |
| 79 | "prompt_tokens", |
| 80 | "output_tokens", |
| 81 | "req_start_time", |
| 82 | "req_finish_time", |
| 83 | "segment_start", |
| 84 | "segment_end", |
| 85 | "duration", |
| 86 | ], |
| 87 | ) |
| 88 | |
| 89 | # Customize hover template to show only time without date |
| 90 | fig.update_traces( |
| 91 | hovertemplate="<b>%{y}</b><br>" |
| 92 | "Type: %{fullData.name}<br>" |
| 93 | "Start: %{customdata[4]}<br>" |
| 94 | "End: %{customdata[5]}<br>" |
| 95 | "Duration: %{customdata[6]}<br>" |
| 96 | "Prompt Tokens: %{customdata[0]}<br>" |
| 97 | "Output Tokens: %{customdata[1]}<br>" |
| 98 | "Request Start Time: %{customdata[2]}<br>" |
| 99 | "Request End Time: %{customdata[3]}<br>" |
| 100 | "<extra></extra>" |
| 101 | ) |
| 102 | |
| 103 | fig.update_yaxes(autorange="reversed") |
| 104 | fig.update_layout( |
| 105 | xaxis_title="Time", |
| 106 | yaxis_title="Request ID", |
| 107 | showlegend=True, |
| 108 | ) |
| 109 | |
| 110 | # Save to HTML |
| 111 | pio.write_html(fig, str(output_path)) |
| 112 | print(f"Timeline plot saved to: {output_path}") |
| 113 | |
| 114 | |
| 115 | def construct_timeline_data( |
| 116 | requests_data: list[dict[str, Any]], |
| 117 | itl_thresholds: list[float], |
| 118 | labels: list[str], |
| 119 | ) -> list[dict[str, Any]]: |
| 120 | """Construct timeline data from request results. |
| 121 | |
| 122 | Args: |
| 123 | requests_data: List of per-request result dictionaries |
| 124 | itl_thresholds: ITL thresholds in seconds |
| 125 | labels: Labels for ITL categories |
| 126 | |
| 127 | Returns: |
| 128 | List of timeline segments for plotting |
| 129 | |
| 130 | """ |
| 131 | |
| 132 | def tostr(sec_time: float) -> str: |
| 133 | """Convert seconds to HH:MM:SS.mmm format.""" |
| 134 | h = int(sec_time // 3600) |
| 135 | assert h < 100, "time seems to last more than 100 hours" |
| 136 | m = int((sec_time % 3600) // 60) |
| 137 | s = sec_time % 60 |
| 138 | return f"{h:02d}:{m:02d}:{s:06.3f}" |
| 139 | |
| 140 | def itl_type(itl: float) -> str: |
| 141 | """Categorize ITL based on thresholds.""" |
| 142 | if itl < itl_thresholds[0]: |
| 143 | return labels[0] |
| 144 | elif itl < itl_thresholds[1]: |
| 145 | return labels[1] |
| 146 | else: |
| 147 | return labels[2] |
| 148 | |
| 149 | # Find the earliest start time to use as t0 |
| 150 | t0 = None |
| 151 | for request in requests_data: |
| 152 | start_time = request.get("start_time") |
| 153 | if start_time is not None and (t0 is None or start_time < t0): |
| 154 | t0 = start_time |
| 155 | |
| 156 | if t0 is None: |
| 157 | return [] |
| 158 | |
| 159 | timeline_data = [] |
| 160 | |
| 161 | for i, request in enumerate(requests_data): |
| 162 | start_time = request.get("start_time") |
| 163 | ttft = request.get("ttft") |
| 164 | itl = request.get("itl", []) |
| 165 | latency = request.get("latency") |
| 166 | prompt_len = request.get("prompt_len", 0) |
| 167 | output_tokens = request.get("output_tokens", 0) |
| 168 | |
| 169 | # Skip requests without required data |
| 170 | if start_time is None or ttft is None or latency is None: |
| 171 | continue |
| 172 | |
| 173 | # Normalize start time |
| 174 | start_time = start_time - t0 |
| 175 | start_time_str = tostr(start_time) |
| 176 | |
| 177 | # TTFT segment |
| 178 | ttft_end = start_time + ttft |
| 179 | ttft_end_str = tostr(ttft_end) |
| 180 | |
| 181 | timeline_data.append( |
| 182 | { |
| 183 | "request_id": f"Req {i}", |
| 184 | "start": start_time_str, |
| 185 | "end": ttft_end_str, |
| 186 | "type": "TTFT", |
| 187 | "prompt_tokens": prompt_len, |
| 188 | "output_tokens": output_tokens, |
| 189 | "req_start_time": tostr(start_time), |
| 190 | "req_finish_time": tostr(start_time + latency), |
| 191 | "segment_start": start_time_str, |
| 192 | "segment_end": ttft_end_str, |
| 193 | "duration": f"{ttft:.3f}s", |
| 194 | } |
| 195 | ) |
| 196 | |
| 197 | # ITL segments |
| 198 | prev_time = ttft_end |
| 199 | prev_time_str = ttft_end_str |
| 200 | |
| 201 | for itl_value in itl: |
| 202 | itl_end = prev_time + itl_value |
| 203 | itl_end_str = tostr(itl_end) |
| 204 | |
| 205 | timeline_data.append( |
| 206 | { |
| 207 | "request_id": f"Req {i}", |
| 208 | "start": prev_time_str, |
| 209 | "end": itl_end_str, |
| 210 | "type": itl_type(itl_value), |
| 211 | "prompt_tokens": prompt_len, |
| 212 | "output_tokens": output_tokens, |
| 213 | "req_start_time": tostr(start_time), |
| 214 | "req_finish_time": tostr(start_time + latency), |
| 215 | "segment_start": prev_time_str, |
| 216 | "segment_end": itl_end_str, |
| 217 | "duration": f"{itl_value:.3f}s", |
| 218 | } |
| 219 | ) |
| 220 | |
| 221 | prev_time = itl_end |
| 222 | prev_time_str = itl_end_str |
| 223 | |
| 224 | return timeline_data |
| 225 | |
| 226 | |
| 227 | def generate_dataset_stats_plot( |
| 228 | results: list[dict[str, Any]], |
| 229 | output_path: Path, |
| 230 | ) -> None: |
| 231 | """Generate a matplotlib figure with dataset statistics. |
| 232 | |
| 233 | Creates a figure with 4 subplots: |
| 234 | - Top-left: Prompt tokens distribution (histogram) |
| 235 | - Top-right: Output tokens distribution (histogram) |
| 236 | - Bottom-left: Prompt+output tokens distribution (histogram) |
| 237 | - Bottom-right: Stacked bar chart (request_id vs tokens) |
| 238 | |
| 239 | Args: |
| 240 | results: List of per-request result dictionaries containing: |
| 241 | - prompt_len: Number of prompt tokens |
| 242 | - output_tokens: Number of output tokens |
| 243 | output_path: Path where the figure will be saved |
| 244 | |
| 245 | """ |
| 246 | # Extract data |
| 247 | prompt_tokens = [] |
| 248 | output_tokens = [] |
| 249 | total_tokens = [] |
| 250 | |
| 251 | for request in results: |
| 252 | prompt_len = request.get("prompt_len", 0) |
| 253 | output_len = request.get("output_tokens", 0) |
| 254 | |
| 255 | prompt_tokens.append(prompt_len) |
| 256 | output_tokens.append(output_len) |
| 257 | total_tokens.append(prompt_len + output_len) |
| 258 | |
| 259 | if not prompt_tokens: |
| 260 | print("No data available for dataset statistics plot") |
| 261 | return |
| 262 | |
| 263 | # Create figure with 4 subplots |
| 264 | fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(14, 10)) |
| 265 | |
| 266 | # Top-left: Prompt tokens distribution |
| 267 | ax1.hist(prompt_tokens, bins=30, color="steelblue", edgecolor="black", alpha=0.7) |
| 268 | ax1.set_xlabel("Prompt Tokens") |
| 269 | ax1.set_ylabel("Frequency") |
| 270 | ax1.set_title("Prompt Tokens Distribution") |
| 271 | ax1.grid(True, alpha=0.3) |
| 272 | |
| 273 | # Top-right: Output tokens distribution |
| 274 | ax2.hist(output_tokens, bins=30, color="coral", edgecolor="black", alpha=0.7) |
| 275 | ax2.set_xlabel("Output Tokens") |
| 276 | ax2.set_ylabel("Frequency") |
| 277 | ax2.set_title("Output Tokens Distribution") |
| 278 | ax2.grid(True, alpha=0.3) |
| 279 | |
| 280 | # Bottom-left: Prompt+output tokens distribution |
| 281 | ax3.hist( |
| 282 | total_tokens, bins=30, color="mediumseagreen", edgecolor="black", alpha=0.7 |
| 283 | ) |
| 284 | ax3.set_xlabel("Total Tokens (Prompt + Output)") |
| 285 | ax3.set_ylabel("Frequency") |
| 286 | ax3.set_title("Total Tokens Distribution") |
| 287 | ax3.grid(True, alpha=0.3) |
| 288 | |
| 289 | # Bottom-right: Stacked bar chart |
| 290 | request_ids = list(range(len(prompt_tokens))) |
| 291 | ax4.bar( |
| 292 | request_ids, prompt_tokens, label="Prompt Tokens", color="steelblue", alpha=0.7 |
| 293 | ) |
| 294 | ax4.bar( |
| 295 | request_ids, |
| 296 | output_tokens, |
| 297 | bottom=prompt_tokens, |
| 298 | label="Output Tokens", |
| 299 | color="coral", |
| 300 | alpha=0.7, |
| 301 | ) |
| 302 | ax4.set_xlabel("Request ID") |
| 303 | ax4.set_ylabel("Tokens") |
| 304 | ax4.set_title("Tokens per Request (Stacked)") |
| 305 | ax4.legend() |
| 306 | ax4.grid(True, alpha=0.3, axis="y") |
| 307 | |
| 308 | # Adjust layout to prevent overlap |
| 309 | plt.tight_layout() |
| 310 | |
| 311 | # Save figure |
| 312 | plt.savefig(str(output_path), dpi=150, bbox_inches="tight") |
| 313 | plt.close(fig) |
| 314 | |
| 315 | print(f"Dataset statistics plot saved to: {output_path}") |