| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import time |
| 5 | |
| 6 | import torch |
| 7 | |
| 8 | from vllm.benchmarks.lib.utils import default_vllm_config |
| 9 | from vllm.model_executor.layers.layernorm import RMSNorm |
| 10 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 11 | from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, set_random_seed |
| 12 | |
| 13 | |
| 14 | @torch.inference_mode() |
| 15 | @default_vllm_config() |
| 16 | def main( |
| 17 | num_tokens: int, |
| 18 | hidden_size: int, |
| 19 | add_residual: bool, |
| 20 | dtype: torch.dtype, |
| 21 | seed: int = 0, |
| 22 | do_profile: bool = False, |
| 23 | num_warmup_iters: int = 5, |
| 24 | num_iters: int = 100, |
| 25 | ) -> None: |
| 26 | set_random_seed(seed) |
| 27 | torch.set_default_device("cuda") |
| 28 | |
| 29 | layer = RMSNorm(hidden_size).to(dtype=dtype) |
| 30 | layer.weight.data.normal_(mean=1.0, std=0.1) |
| 31 | scale = 1 / (2 * hidden_size) |
| 32 | x = torch.randn(num_tokens, hidden_size, dtype=dtype) |
| 33 | x *= scale |
| 34 | residual = torch.randn_like(x) * scale if add_residual else None |
| 35 | |
| 36 | def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float: |
| 37 | torch.accelerator.synchronize() |
| 38 | if profile: |
| 39 | torch.cuda.cudart().cudaProfilerStart() |
| 40 | start_time = time.perf_counter() |
| 41 | |
| 42 | for _ in range(num_iters): |
| 43 | layer(x, residual) |
| 44 | torch.accelerator.synchronize() |
| 45 | |
| 46 | end_time = time.perf_counter() |
| 47 | if profile: |
| 48 | torch.cuda.cudart().cudaProfilerStop() |
| 49 | return (end_time - start_time) / num_iters |
| 50 | |
| 51 | # Warmup. |
| 52 | print("Warming up...") |
| 53 | run_benchmark = run_cuda_benchmark |
| 54 | run_benchmark(num_iters=num_warmup_iters, profile=False) |
| 55 | |
| 56 | # Benchmark. |
| 57 | if do_profile: |
| 58 | latency = run_benchmark(num_iters=1, profile=True) |
| 59 | else: |
| 60 | latency = run_benchmark(num_iters=num_iters, profile=False) |
| 61 | print(f"Kernel running time: {latency * 1000000:.3f} us") |
| 62 | |
| 63 | |
| 64 | if __name__ == "__main__": |
| 65 | parser = FlexibleArgumentParser(description="Benchmark the layernorm kernel.") |
| 66 | parser.add_argument("--num-tokens", type=int, default=4096) |
| 67 | parser.add_argument("--hidden-size", type=int, default=8192) |
| 68 | parser.add_argument("--add-residual", action="store_true") |
| 69 | parser.add_argument( |
| 70 | "--dtype", type=str, choices=["half", "bfloat16", "float"], default="half" |
| 71 | ) |
| 72 | parser.add_argument("--seed", type=int, default=0) |
| 73 | parser.add_argument("--profile", action="store_true") |
| 74 | parser.add_argument("--num-warmup-iters", type=int, default=5) |
| 75 | parser.add_argument( |
| 76 | "--num-iters", |
| 77 | type=int, |
| 78 | default=100, |
| 79 | help="Number of benchmark iterations. " |
| 80 | "If --profile is set, this number is ignored", |
| 81 | ) |
| 82 | |
| 83 | args = parser.parse_args() |
| 84 | print(args) |
| 85 | |
| 86 | main( |
| 87 | num_tokens=args.num_tokens, |
| 88 | hidden_size=args.hidden_size, |
| 89 | add_residual=args.add_residual, |
| 90 | dtype=STR_DTYPE_TO_TORCH_DTYPE[args.dtype], |
| 91 | seed=args.seed, |
| 92 | do_profile=args.profile, |
| 93 | num_warmup_iters=args.num_warmup_iters, |
| 94 | num_iters=args.num_iters, |
| 95 | ) |