A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Kernel Benchmark for `RMSNorm` (#11241)
Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Xiaoyu Zhang <BBuf@users.noreply.github.com>
Roger Wang committed
Dec 16, 2024 at 22:57 UTC
02222a0256f60319f5bcd56d1d036a943d6334f8
1 file changed
+262
benchmarks/kernels/benchmark_rmsnorm.py
new
+262
@@ -0,0 +1,262 @@
1
+import itertools
2
+from typing import Optional, Tuple, Union
3
+
4
+import torch
5
+import triton
6
+from flashinfer.norm import fused_add_rmsnorm, rmsnorm
7
+from torch import nn
8
+
9
+from vllm import _custom_ops as vllm_ops
10
+
11
+
12
+class HuggingFaceRMSNorm(nn.Module):
13
+
14
+ def __init__(self, hidden_size: int, eps: float = 1e-6) -> None:
15
+ super().__init__()
16
+ self.weight = nn.Parameter(torch.ones(hidden_size))
17
+ self.variance_epsilon = eps
18
+
19
+ def forward(
20
+ self,
21
+ x: torch.Tensor,
22
+ residual: Optional[torch.Tensor] = None,
23
+ ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
24
+ orig_dtype = x.dtype
25
+ x = x.to(torch.float32)
26
+ if residual is not None:
27
+ x = x + residual.to(torch.float32)
28
+ residual = x.to(orig_dtype)
29
+
30
+ variance = x.pow(2).mean(dim=-1, keepdim=True)
31
+ x = x * torch.rsqrt(variance + self.variance_epsilon)
32
+ x = x.to(orig_dtype) * self.weight
33
+ if residual is None:
34
+ return x
35
+ else:
36
+ return x, residual
37
+
38
+
39
+def rmsnorm_naive(
40
+ x: torch.Tensor,
41
+ weight: torch.Tensor,
42
+ residual: Optional[torch.Tensor] = None,
43
+ eps: float = 1e-6,
44
+):
45
+ naive_norm = HuggingFaceRMSNorm(x.shape[-1], eps=eps)
46
+ naive_norm.weight = nn.Parameter(weight)
47
+ naive_norm = naive_norm.to(x.device)
48
+
49
+ orig_shape = x.shape
50
+ x = x.view(-1, x.shape[-1])
51
+ if residual is not None:
52
+ residual = residual.view(-1, residual.shape[-1])
53
+
54
+ output = naive_norm(x, residual)
55
+
56
+ if isinstance(output, tuple):
57
+ output = (output[0].view(orig_shape), output[1].view(orig_shape))
58
+ else:
59
+ output = output.view(orig_shape)
60
+ return output
61
+
62
+
63
+def rmsnorm_flashinfer(
64
+ x: torch.Tensor,
65
+ weight: torch.Tensor,
66
+ residual: Optional[torch.Tensor] = None,
67
+ eps: float = 1e-6,
68
+):
69
+ orig_shape = x.shape
70
+ x = x.view(-1, x.shape[-1])
71
+ if residual is not None:
72
+ residual = residual.view(-1, residual.shape[-1])
73
+
74
+ if residual is not None:
75
+ fused_add_rmsnorm(x, residual, weight, eps)
76
+ output = (x, residual)
77
+ else:
78
+ output = rmsnorm(x, weight, eps)
79
+
80
+ if isinstance(output, tuple):
81
+ output = (output[0].view(orig_shape), output[1].view(orig_shape))
82
+ else:
83
+ output = output.view(orig_shape)
84
+ return output
85
+
86
+
87
+def rmsnorm_vllm(
88
+ x: torch.Tensor,
89
+ weight: torch.Tensor,
90
+ residual: Optional[torch.Tensor] = None,
91
+ eps: float = 1e-6,
92
+):
93
+ orig_shape = x.shape
94
+ x = x.view(-1, x.shape[-1])
95
+ if residual is not None:
96
+ residual = residual.view(-1, residual.shape[-1])
97
+
98
+ if residual is not None:
99
+ vllm_ops.fused_add_rms_norm(x, residual, weight, eps)
100
+ output = (x, residual)
101
+ else:
102
+ out = torch.empty_like(x)
103
+ vllm_ops.rms_norm(out, x, weight, eps)
104
+ output = out
105
+
106
+ if isinstance(output, tuple):
107
+ output = (output[0].view(orig_shape), output[1].view(orig_shape))
108
+ else:
109
+ output = output.view(orig_shape)
110
+ return output
111
+
112
+
113
+def calculate_diff(batch_size, seq_len, hidden_size, use_residual=True):
114
+ dtype = torch.bfloat16
115
+ x = torch.randn(batch_size,
116
+ seq_len,
117
+ hidden_size,
118
+ dtype=dtype,
119
+ device="cuda")
120
+ weight = torch.ones(hidden_size, dtype=dtype, device="cuda")
121
+ residual = torch.randn_like(x) if use_residual else None
122
+
123
+ output_naive = rmsnorm_naive(
124
+ x.clone(), weight,
125
+ residual.clone() if residual is not None else None)
126
+ output_flashinfer = rmsnorm_flashinfer(
127
+ x.clone(), weight,
128
+ residual.clone() if residual is not None else None)
129
+ output_vllm = rmsnorm_vllm(
130
+ x.clone(), weight,
131
+ residual.clone() if residual is not None else None)
132
+
133
+ if use_residual:
134
+ output_naive = output_naive[0]
135
+ output_flashinfer = output_flashinfer[0]
136
+ output_vllm = output_vllm[0]
137
+
138
+ print(f"Naive output={output_naive}")
139
+ print(f"FlashInfer output={output_flashinfer}")
140
+ print(f"VLLM output={output_vllm}")
141
+
142
+ if torch.allclose(output_naive, output_flashinfer, atol=1e-2,
143
+ rtol=1e-2) and torch.allclose(
144
+ output_naive, output_vllm, atol=1e-2, rtol=1e-2):
145
+ print("✅ All implementations match")
146
+ else:
147
+ print("❌ Implementations differ")
148
+
149
+
150
+batch_size_range = [2**i for i in range(0, 7, 2)]
151
+seq_length_range = [2**i for i in range(6, 11, 1)]
152
+head_num_range = [32, 48]
153
+configs = list(
154
+ itertools.product(head_num_range, batch_size_range, seq_length_range))
155
+
156
+
157
+def get_benchmark(use_residual):
158
+
159
+ @triton.testing.perf_report(
160
+ triton.testing.Benchmark(
161
+ x_names=["head_num", "batch_size", "seq_len"],
162
+ x_vals=[list(_) for _ in configs],
163
+ line_arg="provider",
164
+ line_vals=["huggingface", "flashinfer", "vllm"],
165
+ line_names=["HuggingFace", "FlashInfer", "vLLM"],
166
+ styles=[("blue", "-"), ("green", "-"), ("red", "-")],
167
+ ylabel="us",
168
+ plot_name=
169
+ f"rmsnorm-perf-{'with' if use_residual else 'without'}-residual",
170
+ args={},
171
+ ))
172
+ def benchmark(head_num, batch_size, seq_len, provider):
173
+ dtype = torch.bfloat16
174
+ hidden_size = head_num * 128 # assuming head_dim = 128
175
+
176
+ x = torch.randn(batch_size,
177
+ seq_len,
178
+ hidden_size,
179
+ dtype=dtype,
180
+ device="cuda")
181
+ weight = torch.ones(hidden_size, dtype=dtype, device="cuda")
182
+ residual = torch.randn_like(x) if use_residual else None
183
+
184
+ quantiles = [0.5, 0.2, 0.8]
185
+
186
+ if provider == "huggingface":
187
+ ms, min_ms, max_ms = triton.testing.do_bench(
188
+ lambda: rmsnorm_naive(
189
+ x.clone(),
190
+ weight,
191
+ residual.clone() if residual is not None else None,
192
+ ),
193
+ quantiles=quantiles,
194
+ )
195
+ elif provider == "flashinfer":
196
+ ms, min_ms, max_ms = triton.testing.do_bench(
197
+ lambda: rmsnorm_flashinfer(
198
+ x.clone(),
199
+ weight,
200
+ residual.clone() if residual is not None else None,
201
+ ),
202
+ quantiles=quantiles,
203
+ )
204
+ else:
205
+ ms, min_ms, max_ms = triton.testing.do_bench(
206
+ lambda: rmsnorm_vllm(
207
+ x.clone(),
208
+ weight,
209
+ residual.clone() if residual is not None else None,
210
+ ),
211
+ quantiles=quantiles,
212
+ )
213
+
214
+ return 1000 * ms, 1000 * max_ms, 1000 * min_ms
215
+
216
+ return benchmark
217
+
218
+
219
+if __name__ == "__main__":
220
+ import argparse
221
+
222
+ parser = argparse.ArgumentParser()
223
+ parser.add_argument(
224
+ "--batch-size",
225
+ type=int,
226
+ default=4,
227
+ help="Batch size",
228
+ )
229
+ parser.add_argument(
230
+ "--seq-len",
231
+ type=int,
232
+ default=128,
233
+ help="Sequence length",
234
+ )
235
+ parser.add_argument(
236
+ "--hidden-size",
237
+ type=int,
238
+ default=4096,
239
+ help="Hidden size (2nd dimension) of the sequence",
240
+ )
241
+ parser.add_argument("--use-residual",
242
+ action="store_true",
243
+ help="Whether to use residual connection")
244
+ parser.add_argument(
245
+ "--save-path",
246
+ type=str,
247
+ default="./configs/rmsnorm/",
248
+ help="Path to save rmsnorm benchmark results",
249
+ )
250
+
251
+ args = parser.parse_args()
252
+
253
+ # Run correctness test
254
+ calculate_diff(batch_size=args.batch_size,
255
+ seq_len=args.seq_len,
256
+ hidden_size=args.hidden_size,
257
+ use_residual=args.use_residual)
258
+
259
+ # Get the benchmark function with proper use_residual setting
260
+ benchmark = get_benchmark(args.use_residual)
261
+ # Run performance benchmark
262
+ benchmark.run(print_data=True, save_path=args.save_path)