| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import dataclasses |
| 5 | from collections.abc import Callable, Iterable |
| 6 | from typing import Any |
| 7 | |
| 8 | import torch |
| 9 | import torch.utils.benchmark as TBenchmark |
| 10 | from torch.utils.benchmark import Measurement as TMeasurement |
| 11 | |
| 12 | |
| 13 | @dataclasses.dataclass |
| 14 | class CudaGraphBenchParams: |
| 15 | num_ops_in_cuda_graph: int |
| 16 | |
| 17 | |
| 18 | @dataclasses.dataclass |
| 19 | class ArgPool: |
| 20 | """When some argument of the benchmarking function is annotated with this type, |
| 21 | the benchmarking class (BenchMM) will collapse the argument to a pick a |
| 22 | single value from the given list of values, during function invocation. |
| 23 | For every invocation during a benchmarking run, it will choose a |
| 24 | different value from the list. |
| 25 | """ |
| 26 | |
| 27 | values: Iterable[Any] |
| 28 | |
| 29 | def __getitem__(self, index): |
| 30 | return self.values[index] |
| 31 | |
| 32 | |
| 33 | class Bench: |
| 34 | class ArgsIterator: |
| 35 | def __init__(self, args_list, kwargs_list): |
| 36 | assert len(args_list) == len(kwargs_list) |
| 37 | self.args_list = args_list |
| 38 | self.kwargs_list = kwargs_list |
| 39 | self.n = len(self.args_list) |
| 40 | self.idx = 0 |
| 41 | |
| 42 | def __next__(self): |
| 43 | while True: |
| 44 | yield (self.args_list[self.idx], self.kwargs_list[self.idx]) |
| 45 | self.idx += 1 |
| 46 | self.idx = self.idx % self.n |
| 47 | |
| 48 | def reset(self): |
| 49 | self.idx = 0 |
| 50 | |
| 51 | @property |
| 52 | def n_args(self): |
| 53 | return self.n |
| 54 | |
| 55 | def __init__( |
| 56 | self, |
| 57 | cuda_graph_params: CudaGraphBenchParams | None, |
| 58 | label: str, |
| 59 | sub_label: str, |
| 60 | description: str, |
| 61 | fn: Callable, |
| 62 | *args, |
| 63 | **kwargs, |
| 64 | ): |
| 65 | self.cuda_graph_params = cuda_graph_params |
| 66 | self.use_cuda_graph = self.cuda_graph_params is not None |
| 67 | self.label = label |
| 68 | self.sub_label = sub_label |
| 69 | self.description = description |
| 70 | self.fn = fn |
| 71 | |
| 72 | # Process args |
| 73 | self._args = args |
| 74 | self._kwargs = kwargs |
| 75 | self.args_list, self.kwargs_list = self.collapse_argpool(*args, **kwargs) |
| 76 | self.args_iterator = self.ArgsIterator(self.args_list, self.kwargs_list) |
| 77 | |
| 78 | # Cudagraph runner |
| 79 | self.g = None |
| 80 | if self.use_cuda_graph: |
| 81 | self.g = self.get_cuda_graph_runner() |
| 82 | |
| 83 | # benchmark run params |
| 84 | self.min_run_time = 1 |
| 85 | |
| 86 | def collapse_argpool(self, *args, **kwargs): |
| 87 | argpool_args = [arg for arg in args if isinstance(arg, ArgPool)] + [ |
| 88 | arg for arg in kwargs.values() if isinstance(arg, ArgPool) |
| 89 | ] |
| 90 | if len(argpool_args) == 0: |
| 91 | return [args], [kwargs] |
| 92 | |
| 93 | # Make sure all argpools are of the same size |
| 94 | argpool_size = len(argpool_args[0].values) |
| 95 | assert all([argpool_size == len(arg.values) for arg in argpool_args]) |
| 96 | |
| 97 | # create copies of the args |
| 98 | args_list = [] |
| 99 | kwargs_list = [] |
| 100 | for _ in range(argpool_size): |
| 101 | args_list.append(args) |
| 102 | kwargs_list.append(kwargs.copy()) |
| 103 | |
| 104 | for i in range(argpool_size): |
| 105 | # collapse args; Just pick the ith value |
| 106 | args_list[i] = tuple( |
| 107 | [arg[i] if isinstance(arg, ArgPool) else arg for arg in args_list[i]] |
| 108 | ) |
| 109 | |
| 110 | # collapse kwargs |
| 111 | kwargs_i = kwargs_list[i] |
| 112 | arg_pool_keys = [k for k, v in kwargs_i.items() if isinstance(v, ArgPool)] |
| 113 | for k in arg_pool_keys: |
| 114 | # again just pick the ith value |
| 115 | kwargs_i[k] = kwargs_i[k][i] |
| 116 | kwargs_list[i] = kwargs_i |
| 117 | |
| 118 | return args_list, kwargs_list |
| 119 | |
| 120 | def get_cuda_graph_runner(self): |
| 121 | assert self.use_cuda_graph |
| 122 | assert self.args_iterator is not None |
| 123 | |
| 124 | num_graph_ops = self.cuda_graph_params.num_ops_in_cuda_graph |
| 125 | |
| 126 | # warmup |
| 127 | args_it = self.args_iterator.__next__() |
| 128 | for _ in range(2): |
| 129 | args, kwargs = next(args_it) |
| 130 | self.fn(*args, **kwargs) |
| 131 | |
| 132 | self.args_iterator.reset() |
| 133 | args_it = self.args_iterator.__next__() |
| 134 | stream = torch.cuda.Stream() |
| 135 | with torch.cuda.stream(stream): |
| 136 | g = torch.cuda.CUDAGraph() |
| 137 | with torch.cuda.graph(g): |
| 138 | for _ in range(num_graph_ops): |
| 139 | args, kwargs = next(args_it) |
| 140 | self.fn(*args, **kwargs) |
| 141 | return g |
| 142 | |
| 143 | def run_cudagrah(self) -> TMeasurement: |
| 144 | assert self.use_cuda_graph |
| 145 | globals = {"g": self.g} |
| 146 | |
| 147 | return TBenchmark.Timer( |
| 148 | stmt="g.replay()", |
| 149 | globals=globals, |
| 150 | label=( |
| 151 | f"{self.label}" |
| 152 | f" | cugraph {self.cuda_graph_params.num_ops_in_cuda_graph} ops" |
| 153 | ), |
| 154 | sub_label=self.sub_label, |
| 155 | description=self.description, |
| 156 | ).blocked_autorange(min_run_time=self.min_run_time) |
| 157 | |
| 158 | def run_eager(self) -> TMeasurement: |
| 159 | setup = None |
| 160 | stmt = None |
| 161 | globals = None |
| 162 | |
| 163 | has_arg_pool = self.args_iterator.n_args > 1 |
| 164 | if has_arg_pool: |
| 165 | setup = """ |
| 166 | args_iterator.reset() |
| 167 | args_it = args_iterator.__next__() |
| 168 | """ |
| 169 | stmt = """ |
| 170 | args, kwargs = next(args_it) |
| 171 | fn(*args, **kwargs) |
| 172 | """ |
| 173 | globals = {"fn": self.fn, "args_iterator": self.args_iterator} |
| 174 | else: |
| 175 | # no arg pool. Just use the args and kwargs directly |
| 176 | self.args_iterator.reset() |
| 177 | args_it = self.args_iterator.__next__() |
| 178 | args, kwargs = next(args_it) |
| 179 | |
| 180 | setup = "" |
| 181 | stmt = """ |
| 182 | fn(*args, **kwargs) |
| 183 | """ |
| 184 | globals = {"fn": self.fn, "args": args, "kwargs": kwargs} |
| 185 | |
| 186 | return TBenchmark.Timer( |
| 187 | stmt=stmt, |
| 188 | setup=setup, |
| 189 | globals=globals, |
| 190 | label=self.label, |
| 191 | sub_label=self.sub_label, |
| 192 | description=self.description, |
| 193 | ).blocked_autorange(min_run_time=self.min_run_time) |
| 194 | |
| 195 | def run(self) -> TMeasurement: |
| 196 | timer = None |
| 197 | if self.use_cuda_graph: # noqa SIM108 |
| 198 | timer = self.run_cudagrah() |
| 199 | else: |
| 200 | timer = self.run_eager() |
| 201 | if not timer.meets_confidence() or timer.has_warnings: |
| 202 | print("Doesn't meet confidence - re-running bench ...") |
| 203 | return self.run() |
| 204 | return timer |
| 205 | |
| 206 | def __enter__(self): |
| 207 | return self |
| 208 | |
| 209 | def __exit__(self, exc_type, exc_value, traceback): |
| 210 | if exc_type: |
| 211 | print(f"exc type {exc_type}") |
| 212 | print(f"exc value {exc_value}") |
| 213 | print(f"exc traceback {traceback}") |