| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import itertools |
| 5 | |
| 6 | import torch |
| 7 | |
| 8 | import vllm.kernels # noqa: F401 |
| 9 | from vllm import ir |
| 10 | from vllm.benchmarks.lib.utils import default_vllm_config |
| 11 | from vllm.triton_utils import triton |
| 12 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 13 | from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, set_random_seed |
| 14 | |
| 15 | NUM_TOKENS = [1, 2, 8, 32, 128, 512, 2048] |
| 16 | INTERMEDIATE_SIZES = [2048, 4096, 8192, 13824, 16384, 32768] |
| 17 | CONFIGS = list(itertools.product(NUM_TOKENS, INTERMEDIATE_SIZES)) |
| 18 | STD_MULTIPLIER = 1.6448536269514722 |
| 19 | |
| 20 | |
| 21 | @default_vllm_config() |
| 22 | def benchmark_gelu_and_mul_sparse( |
| 23 | num_tokens: int, |
| 24 | intermediate_size: int, |
| 25 | provider: str, |
| 26 | dtype: torch.dtype, |
| 27 | ): |
| 28 | set_random_seed(0) |
| 29 | torch.set_default_device("cuda") |
| 30 | x = torch.randn(num_tokens, 2 * intermediate_size, dtype=dtype) |
| 31 | native = ir.ops.gelu_and_mul_sparse.impls["native"].impl_fn |
| 32 | triton_impl = ir.ops.gelu_and_mul_sparse.impls["triton"].impl_fn |
| 33 | args = (x, STD_MULTIPLIER, "tanh") |
| 34 | |
| 35 | expected = native(*args) |
| 36 | actual = triton_impl(*args) |
| 37 | torch.testing.assert_close( |
| 38 | actual, |
| 39 | expected, |
| 40 | **ir.ops.gelu_and_mul_sparse.get_tolerance(actual.dtype), |
| 41 | ) |
| 42 | |
| 43 | if provider == "triton": |
| 44 | fn = lambda: triton_impl(*args) |
| 45 | elif provider == "native_compiled": |
| 46 | compiled = torch.compile(native, fullgraph=True) |
| 47 | compiled(*args) |
| 48 | fn = lambda: compiled(*args) |
| 49 | else: |
| 50 | fn = lambda: native(*args) |
| 51 | |
| 52 | ms, min_ms, max_ms = triton.testing.do_bench_cudagraph( |
| 53 | fn, quantiles=[0.5, 0.2, 0.8] |
| 54 | ) |
| 55 | return ms, max_ms, min_ms |
| 56 | |
| 57 | |
| 58 | if __name__ == "__main__": |
| 59 | parser = FlexibleArgumentParser( |
| 60 | description="Benchmark the Gemma3n sparse GELU activation." |
| 61 | ) |
| 62 | parser.add_argument( |
| 63 | "--dtype", |
| 64 | type=str, |
| 65 | choices=["half", "bfloat16", "float"], |
| 66 | default="bfloat16", |
| 67 | ) |
| 68 | args = parser.parse_args() |
| 69 | dtype = STR_DTYPE_TO_TORCH_DTYPE[args.dtype] |
| 70 | |
| 71 | perf_report = triton.testing.perf_report( |
| 72 | triton.testing.Benchmark( |
| 73 | x_names=["num_tokens", "intermediate_size"], |
| 74 | x_vals=CONFIGS, |
| 75 | line_arg="provider", |
| 76 | line_vals=["triton", "native_compiled", "native_eager"], |
| 77 | line_names=["Triton", "Native (torch.compile)", "Native (eager)"], |
| 78 | styles=[("blue", "-"), ("green", "-"), ("red", "-")], |
| 79 | ylabel="ms", |
| 80 | plot_name="gelu-and-mul-sparse-performance", |
| 81 | args={}, |
| 82 | ) |
| 83 | ) |
| 84 | perf_report( |
| 85 | lambda num_tokens, intermediate_size, provider: ( |
| 86 | benchmark_gelu_and_mul_sparse( |
| 87 | num_tokens, intermediate_size, provider, dtype |
| 88 | ) |
| 89 | ) |
| 90 | ).run(print_data=True) |