@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e1054247ba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[Optimization] Implement fused add rmsnorm (#1667)

ljss committed Nov 19, 2023 at 10:18 UTC e1054247ba28aed104a1a26bbca86bac70d642af
9 files changed +166 -61
csrc/layernorm.cpp
+10
@@ -6,9 +6,19 @@ void rms_norm(
6 torch::Tensor& weight,
7 float epsilon);
8
9 +void fused_add_rms_norm(
10 + torch::Tensor& input,
11 + torch::Tensor& residual,
12 + torch::Tensor& weight,
13 + float epsilon);
14 +
15 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
16 m.def(
17 "rms_norm",
18 &rms_norm,
19 "Apply Root Mean Square (RMS) Normalization to the input tensor.");
20 + m.def(
21 + "fused_add_rms_norm",
22 + &fused_add_rms_norm,
23 + "In-place fused Add and RMS Normalization");
24 }
csrc/layernorm_kernels.cu
+55
@@ -34,6 +34,36 @@ __global__ void rms_norm_kernel(
34 }
35 }
36
37 +// TODO: Further optimize this kernel.
38 +template<typename scalar_t>
39 +__global__ void fused_add_rms_norm_kernel(
40 + scalar_t* __restrict__ input, // [..., hidden_size]
41 + scalar_t* __restrict__ residual, // [..., hidden_size]
42 + const scalar_t* __restrict__ weight, // [hidden_size]
43 + const float epsilon,
44 + const int num_tokens,
45 + const int hidden_size) {
46 + __shared__ float s_variance;
47 + float variance = 0.0f;
48 +
49 + for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
50 + float x = (float) input[blockIdx.x * hidden_size + idx];
51 + x += (float) residual[blockIdx.x * hidden_size + idx];
52 + variance += x * x;
53 + residual[blockIdx.x * hidden_size + idx] = (scalar_t) x;
54 + }
55 + variance = blockReduceSum<float>(variance);
56 + if (threadIdx.x == 0) {
57 + s_variance = rsqrtf(variance / hidden_size + epsilon);
58 + }
59 + __syncthreads();
60 +
61 + for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
62 + float x = (float) residual[blockIdx.x * hidden_size + idx];
63 + input[blockIdx.x * hidden_size + idx] = ((scalar_t) (x * s_variance)) * weight[idx];
64 + }
65 +}
66 +
67 } // namespace vllm
68
69 void rms_norm(
@@ -60,3 +90,28 @@ void rms_norm(
90 hidden_size);
91 });
92 }
93 +
94 +void fused_add_rms_norm(
95 + torch::Tensor& input, // [..., hidden_size]
96 + torch::Tensor& residual, // [..., hidden_size]
97 + torch::Tensor& weight, // [hidden_size]
98 + float epsilon) {
99 + int hidden_size = input.size(-1);
100 + int num_tokens = input.numel() / hidden_size;
101 +
102 + dim3 grid(num_tokens);
103 + dim3 block(std::min(hidden_size, 1024));
104 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
105 + VLLM_DISPATCH_FLOATING_TYPES(
106 + input.scalar_type(),
107 + "fused_add_rms_norm_kernel",
108 + [&] {
109 + vllm::fused_add_rms_norm_kernel<scalar_t><<<grid, block, 0, stream>>>(
110 + input.data_ptr<scalar_t>(),
111 + residual.data_ptr<scalar_t>(),
112 + weight.data_ptr<scalar_t>(),
113 + epsilon,
114 + num_tokens,
115 + hidden_size);
116 + });
117 +}
vllm/model_executor/layers/layernorm.py
+15 -1
@@ -1,4 +1,6 @@
1 """Custom normalization layers."""
2 +from typing import Optional, Tuple, Union
3 +
4 import torch
5 import torch.nn as nn
6
@@ -21,7 +23,19 @@ class RMSNorm(nn.Module):
23 self.weight = nn.Parameter(torch.ones(hidden_size))
24 self.variance_epsilon = eps
25
24 - def forward(self, x: torch.Tensor) -> torch.Tensor:
26 + def forward(
27 + self,
28 + x: torch.Tensor,
29 + residual: Optional[torch.Tensor] = None,
30 + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
31 + if residual is not None:
32 + layernorm_ops.fused_add_rms_norm(
33 + x,
34 + residual,
35 + self.weight.data,
36 + self.variance_epsilon,
37 + )
38 + return x, residual
39 out = torch.empty_like(x)
40 layernorm_ops.rms_norm(
41 out,
vllm/model_executor/models/baichuan.py
+15 -10
@@ -225,10 +225,15 @@ class BaiChuanDecoderLayer(nn.Module):
225 kv_cache: KVCache,
226 input_metadata: InputMetadata,
227 cache_event: Optional[torch.cuda.Event],
228 - ) -> torch.Tensor:
228 + residual: Optional[torch.Tensor],
229 + ) -> Tuple[torch.Tensor, torch.Tensor]:
230 # Self Attention
230 - residual = hidden_states
231 - hidden_states = self.input_layernorm(hidden_states)
231 + if residual is None:
232 + residual = hidden_states
233 + hidden_states = self.input_layernorm(hidden_states)
234 + else:
235 + hidden_states, residual = self.input_layernorm(
236 + hidden_states, residual)
237 hidden_states = self.self_attn(
238 positions=positions,
239 hidden_states=hidden_states,
@@ -236,14 +241,12 @@ class BaiChuanDecoderLayer(nn.Module):
241 input_metadata=input_metadata,
242 cache_event=cache_event,
243 )
239 - hidden_states = residual + hidden_states
244
245 # Fully Connected
242 - residual = hidden_states
243 - hidden_states = self.post_attention_layernorm(hidden_states)
246 + hidden_states, residual = self.post_attention_layernorm(
247 + hidden_states, residual)
248 hidden_states = self.mlp(hidden_states)
245 - hidden_states = residual + hidden_states
246 - return hidden_states
249 + return hidden_states, residual
250
251
252 class BaiChuanModel(nn.Module):
@@ -276,20 +279,22 @@ class BaiChuanModel(nn.Module):
279 cache_events: Optional[List[torch.cuda.Event]],
280 ) -> torch.Tensor:
281 hidden_states = self.embed_tokens(input_ids)
282 + residual = None
283 for i in range(len(self.layers)):
284 if cache_events is None:
285 cache_event = None
286 else:
287 cache_event = cache_events[i]
288 layer = self.layers[i]
285 - hidden_states = layer(
289 + hidden_states, residual = layer(
290 positions,
291 hidden_states,
292 kv_caches[i],
293 input_metadata,
294 cache_event,
295 + residual,
296 )
292 - hidden_states = self.norm(hidden_states)
297 + hidden_states, _ = self.norm(hidden_states, residual)
298 return hidden_states
299
300
vllm/model_executor/models/internlm.py
+15 -10
@@ -155,10 +155,15 @@ class InternLMDecoderLayer(nn.Module):
155 kv_cache: KVCache,
156 input_metadata: InputMetadata,
157 cache_event: Optional[torch.cuda.Event],
158 - ) -> torch.Tensor:
158 + residual: Optional[torch.Tensor],
159 + ) -> Tuple[torch.Tensor, torch.Tensor]:
160 # Self Attention
160 - residual = hidden_states
161 - hidden_states = self.input_layernorm(hidden_states)
161 + if residual is None:
162 + residual = hidden_states
163 + hidden_states = self.input_layernorm(hidden_states)
164 + else:
165 + hidden_states, residual = self.input_layernorm(
166 + hidden_states, residual)
167 hidden_states = self.self_attn(
168 positions=positions,
169 hidden_states=hidden_states,
@@ -166,14 +171,12 @@ class InternLMDecoderLayer(nn.Module):
171 input_metadata=input_metadata,
172 cache_event=cache_event,
173 )
169 - hidden_states = residual + hidden_states
174
175 # Fully Connected
172 - residual = hidden_states
173 - hidden_states = self.post_attention_layernorm(hidden_states)
176 + hidden_states, residual = self.post_attention_layernorm(
177 + hidden_states, residual)
178 hidden_states = self.mlp(hidden_states)
175 - hidden_states = residual + hidden_states
176 - return hidden_states
179 + return hidden_states, residual
180
181
182 class InternLMModel(nn.Module):
@@ -208,20 +211,22 @@ class InternLMModel(nn.Module):
211 cache_events: Optional[List[torch.cuda.Event]],
212 ) -> torch.Tensor:
213 hidden_states = self.embed_tokens(input_ids)
214 + residual = None
215 for i in range(len(self.layers)):
216 if cache_events is None:
217 cache_event = None
218 else:
219 cache_event = cache_events[i]
220 layer = self.layers[i]
217 - hidden_states = layer(
221 + hidden_states, residual = layer(
222 positions,
223 hidden_states,
224 kv_caches[i],
225 input_metadata,
226 cache_event,
227 + residual,
228 )
224 - hidden_states = self.norm(hidden_states)
229 + hidden_states, _ = self.norm(hidden_states, residual)
230 return hidden_states
231
232
vllm/model_executor/models/llama.py
+15 -10
@@ -197,10 +197,15 @@ class LlamaDecoderLayer(nn.Module):
197 kv_cache: KVCache,
198 input_metadata: InputMetadata,
199 cache_event: Optional[torch.cuda.Event],
200 - ) -> torch.Tensor:
200 + residual: Optional[torch.Tensor],
201 + ) -> Tuple[torch.Tensor, torch.Tensor]:
202 # Self Attention
202 - residual = hidden_states
203 - hidden_states = self.input_layernorm(hidden_states)
203 + if residual is None:
204 + residual = hidden_states
205 + hidden_states = self.input_layernorm(hidden_states)
206 + else:
207 + hidden_states, residual = self.input_layernorm(
208 + hidden_states, residual)
209 hidden_states = self.self_attn(
210 positions=positions,
211 hidden_states=hidden_states,
@@ -208,14 +213,12 @@ class LlamaDecoderLayer(nn.Module):
213 input_metadata=input_metadata,
214 cache_event=cache_event,
215 )
211 - hidden_states = residual + hidden_states
216
217 # Fully Connected
214 - residual = hidden_states
215 - hidden_states = self.post_attention_layernorm(hidden_states)
218 + hidden_states, residual = self.post_attention_layernorm(
219 + hidden_states, residual)
220 hidden_states = self.mlp(hidden_states)
217 - hidden_states = residual + hidden_states
218 - return hidden_states
221 + return hidden_states, residual
222
223
224 class LlamaModel(nn.Module):
@@ -248,20 +251,22 @@ class LlamaModel(nn.Module):
251 cache_events: Optional[List[torch.cuda.Event]],
252 ) -> torch.Tensor:
253 hidden_states = self.embed_tokens(input_ids)
254 + residual = None
255 for i in range(len(self.layers)):
256 if cache_events is None:
257 cache_event = None
258 else:
259 cache_event = cache_events[i]
260 layer = self.layers[i]
257 - hidden_states = layer(
261 + hidden_states, residual = layer(
262 positions,
263 hidden_states,
264 kv_caches[i],
265 input_metadata,
266 cache_event,
267 + residual,
268 )
264 - hidden_states = self.norm(hidden_states)
269 + hidden_states, _ = self.norm(hidden_states, residual)
270 return hidden_states
271
272
vllm/model_executor/models/mistral.py
+15 -10
@@ -191,10 +191,15 @@ class MistralDecoderLayer(nn.Module):
191 kv_cache: KVCache,
192 input_metadata: InputMetadata,
193 cache_event: Optional[torch.cuda.Event],
194 - ) -> torch.Tensor:
194 + residual: Optional[torch.Tensor],
195 + ) -> Tuple[torch.Tensor, torch.Tensor]:
196 # Self Attention
196 - residual = hidden_states
197 - hidden_states = self.input_layernorm(hidden_states)
197 + if residual is None:
198 + residual = hidden_states
199 + hidden_states = self.input_layernorm(hidden_states)
200 + else:
201 + hidden_states, residual = self.input_layernorm(
202 + hidden_states, residual)
203 hidden_states = self.self_attn(
204 positions=positions,
205 hidden_states=hidden_states,
@@ -202,14 +207,12 @@ class MistralDecoderLayer(nn.Module):
207 input_metadata=input_metadata,
208 cache_event=cache_event,
209 )
205 - hidden_states = residual + hidden_states
210
211 # Fully Connected
208 - residual = hidden_states
209 - hidden_states = self.post_attention_layernorm(hidden_states)
212 + hidden_states, residual = self.post_attention_layernorm(
213 + hidden_states, residual)
214 hidden_states = self.mlp(hidden_states)
211 - hidden_states = residual + hidden_states
212 - return hidden_states
215 + return hidden_states, residual
216
217
218 class MistralModel(nn.Module):
@@ -243,20 +246,22 @@ class MistralModel(nn.Module):
246 cache_events: Optional[List[torch.cuda.Event]],
247 ) -> torch.Tensor:
248 hidden_states = self.embed_tokens(input_ids)
249 + residual = None
250 for i in range(len(self.layers)):
251 if cache_events is None:
252 cache_event = None
253 else:
254 cache_event = cache_events[i]
255 layer = self.layers[i]
252 - hidden_states = layer(
256 + hidden_states, residual = layer(
257 positions,
258 hidden_states,
259 kv_caches[i],
260 input_metadata,
261 cache_event,
262 + residual,
263 )
259 - hidden_states = self.norm(hidden_states)
264 + hidden_states, _ = self.norm(hidden_states, residual)
265 return hidden_states
266
267
vllm/model_executor/models/qwen.py
+13 -10
@@ -159,10 +159,14 @@ class QWenBlock(nn.Module):
159 kv_cache: KVCache,
160 input_metadata: InputMetadata,
161 cache_event: Optional[torch.cuda.Event],
162 - ) -> torch.Tensor:
162 + residual: Optional[torch.Tensor],
163 + ) -> Tuple[torch.Tensor, torch.Tensor]:
164 # Self Attention
164 - residual = hidden_states
165 - hidden_states = self.ln_1(hidden_states)
165 + if residual is None:
166 + residual = hidden_states
167 + hidden_states = self.ln_1(hidden_states)
168 + else:
169 + hidden_states, residual = self.ln_1(hidden_states, residual)
170 hidden_states = self.attn(
171 positions=positions,
172 hidden_states=hidden_states,
@@ -170,14 +174,11 @@ class QWenBlock(nn.Module):
174 input_metadata=input_metadata,
175 cache_event=cache_event,
176 )
173 - hidden_states = residual + hidden_states
177
178 # Fully Connected
176 - residual = hidden_states
177 - hidden_states = self.ln_2(hidden_states)
179 + hidden_states, residual = self.ln_2(hidden_states, residual)
180 hidden_states = self.mlp(hidden_states)
179 - hidden_states = residual + hidden_states
180 - return hidden_states
181 + return hidden_states, residual
182
183
184 class QWenModel(nn.Module):
@@ -210,20 +211,22 @@ class QWenModel(nn.Module):
211 cache_events: Optional[List[torch.cuda.Event]],
212 ) -> torch.Tensor:
213 hidden_states = self.wte(input_ids)
214 + residual = None
215 for i in range(len(self.h)):
216 if cache_events is None:
217 cache_event = None
218 else:
219 cache_event = cache_events[i]
220 layer = self.h[i]
219 - hidden_states = layer(
221 + hidden_states, residual = layer(
222 positions,
223 hidden_states,
224 kv_caches[i],
225 input_metadata,
226 cache_event,
227 + residual,
228 )
226 - hidden_states = self.ln_f(hidden_states)
229 + hidden_states, _ = self.ln_f(hidden_states, residual)
230 return hidden_states
231
232
vllm/model_executor/models/yi.py
+13 -10
@@ -195,10 +195,14 @@ class YiDecoderLayer(nn.Module):
195 kv_cache: KVCache,
196 input_metadata: InputMetadata,
197 cache_event: Optional[torch.cuda.Event],
198 - ) -> torch.Tensor:
198 + residual: Optional[torch.Tensor],
199 + ) -> Tuple[torch.Tensor, torch.Tensor]:
200 # Self Attention
200 - residual = hidden_states
201 - hidden_states = self.ln1(hidden_states)
201 + if residual is None:
202 + residual = hidden_states
203 + hidden_states = self.ln1(hidden_states)
204 + else:
205 + hidden_states, residual = self.ln1(hidden_states, residual)
206 hidden_states = self.self_attn(
207 positions=positions,
208 hidden_states=hidden_states,
@@ -206,14 +210,11 @@ class YiDecoderLayer(nn.Module):
210 input_metadata=input_metadata,
211 cache_event=cache_event,
212 )
209 - hidden_states = residual + hidden_states
213
214 # Fully Connected
212 - residual = hidden_states
213 - hidden_states = self.ln2(hidden_states)
215 + hidden_states, residual = self.ln2(hidden_states, residual)
216 hidden_states = self.mlp(hidden_states)
215 - hidden_states = residual + hidden_states
216 - return hidden_states
217 + return hidden_states, residual
218
219
220 class YiModel(nn.Module):
@@ -246,20 +247,22 @@ class YiModel(nn.Module):
247 cache_events: Optional[List[torch.cuda.Event]],
248 ) -> torch.Tensor:
249 hidden_states = self.embed_tokens(input_ids)
250 + residual = None
251 for i in range(len(self.layers)):
252 if cache_events is None:
253 cache_event = None
254 else:
255 cache_event = cache_events[i]
256 layer = self.layers[i]
255 - hidden_states = layer(
257 + hidden_states, residual = layer(
258 positions,
259 hidden_states,
260 kv_caches[i],
261 input_metadata,
262 cache_event,
263 + residual,
264 )
262 - hidden_states = self.norm(hidden_states)
265 + hidden_states, _ = self.norm(hidden_states, residual)
266 return hidden_states
267
268