| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import time |
| 5 | |
| 6 | import torch |
| 7 | |
| 8 | from vllm import _custom_ops as ops |
| 9 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 10 | from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, set_random_seed |
| 11 | |
| 12 | |
| 13 | @torch.inference_mode() |
| 14 | def main( |
| 15 | num_tokens: int, |
| 16 | hidden_size: int, |
| 17 | static_scale: bool, |
| 18 | quant_dtype: torch.dtype, |
| 19 | dtype: torch.dtype, |
| 20 | seed: int = 0, |
| 21 | do_profile: bool = False, |
| 22 | num_warmup_iters: int = 5, |
| 23 | num_iters: int = 100, |
| 24 | ) -> None: |
| 25 | set_random_seed(seed) |
| 26 | torch.set_default_device("cuda") |
| 27 | |
| 28 | x = torch.randn(num_tokens, hidden_size, dtype=dtype) |
| 29 | scale = torch.randn(1, 1, dtype=torch.float32) if static_scale else None |
| 30 | |
| 31 | def run_cuda_benchmark(num_iters: int, profile: bool = False) -> float: |
| 32 | torch.accelerator.synchronize() |
| 33 | if profile: |
| 34 | torch.cuda.cudart().cudaProfilerStart() |
| 35 | start_time = time.perf_counter() |
| 36 | |
| 37 | for _ in range(num_iters): |
| 38 | if quant_dtype == torch.int8: |
| 39 | ops.scaled_int8_quant(x, scale) |
| 40 | else: |
| 41 | ops.scaled_fp8_quant(x, scale) |
| 42 | torch.accelerator.synchronize() |
| 43 | |
| 44 | end_time = time.perf_counter() |
| 45 | if profile: |
| 46 | torch.cuda.cudart().cudaProfilerStop() |
| 47 | return (end_time - start_time) / num_iters |
| 48 | |
| 49 | # Warmup. |
| 50 | print("Warming up...") |
| 51 | run_benchmark = run_cuda_benchmark |
| 52 | run_benchmark(num_iters=num_warmup_iters, profile=False) |
| 53 | |
| 54 | # Benchmark. |
| 55 | if do_profile: |
| 56 | latency = run_benchmark(num_iters=1, profile=True) |
| 57 | else: |
| 58 | latency = run_benchmark(num_iters=num_iters, profile=False) |
| 59 | print(f"Kernel running time: {latency * 1000000:.3f} us") |
| 60 | |
| 61 | |
| 62 | if __name__ == "__main__": |
| 63 | |
| 64 | def to_torch_dtype(dt): |
| 65 | if dt == "int8": |
| 66 | return torch.int8 |
| 67 | if dt == "fp8": |
| 68 | return torch.float8_e4m3fn |
| 69 | raise ValueError(f"Unsupported dtype: {dt}") |
| 70 | |
| 71 | parser = FlexibleArgumentParser( |
| 72 | description="Benchmark the quantization (fp8 or int8) kernel." |
| 73 | ) |
| 74 | parser.add_argument("--num-tokens", type=int, default=4096) |
| 75 | parser.add_argument("--hidden-size", type=int, default=8192) |
| 76 | parser.add_argument("--static-scale", action="store_true") |
| 77 | parser.add_argument( |
| 78 | "--quant-dtype", type=str, choices=["fp8", "int8"], default="int8" |
| 79 | ) |
| 80 | parser.add_argument( |
| 81 | "--dtype", type=str, choices=["half", "bfloat16", "float"], default="half" |
| 82 | ) |
| 83 | |
| 84 | parser.add_argument("--seed", type=int, default=0) |
| 85 | parser.add_argument("--profile", action="store_true") |
| 86 | parser.add_argument("--num-warmup-iters", type=int, default=5) |
| 87 | parser.add_argument( |
| 88 | "--num-iters", |
| 89 | type=int, |
| 90 | default=100, |
| 91 | help="Number of benchmark iterations. " |
| 92 | "If --profile is set, this number is ignored", |
| 93 | ) |
| 94 | |
| 95 | args = parser.parse_args() |
| 96 | print(args) |
| 97 | |
| 98 | main( |
| 99 | num_tokens=args.num_tokens, |
| 100 | hidden_size=args.hidden_size, |
| 101 | static_scale=args.static_scale, |
| 102 | quant_dtype=to_torch_dtype(args.quant_dtype), |
| 103 | dtype=STR_DTYPE_TO_TORCH_DTYPE[args.dtype], |
| 104 | seed=args.seed, |
| 105 | do_profile=args.profile, |
| 106 | num_warmup_iters=args.num_warmup_iters, |
| 107 | num_iters=args.num_iters, |
| 108 | ) |