A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[Optimization] Implement fused add rmsnorm (#1667)
ljss committed
Nov 19, 2023 at 10:18 UTC
e1054247ba28aed104a1a26bbca86bac70d642af
9 files changed
+166
-61
csrc/layernorm.cpp
+10
@@ -6,9 +6,19 @@ void rms_norm(
6
torch::Tensor& weight,
7
float epsilon);
8
9
+void fused_add_rms_norm(
10
+ torch::Tensor& input,
11
+ torch::Tensor& residual,
12
+ torch::Tensor& weight,
13
+ float epsilon);
14
+
15
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
16
m.def(
17
"rms_norm",
18
&rms_norm,
19
"Apply Root Mean Square (RMS) Normalization to the input tensor.");
20
+ m.def(
21
+ "fused_add_rms_norm",
22
+ &fused_add_rms_norm,
23
+ "In-place fused Add and RMS Normalization");
24
}
csrc/layernorm_kernels.cu
+55
@@ -34,6 +34,36 @@ __global__ void rms_norm_kernel(
34
}
35
}
36
37
+// TODO: Further optimize this kernel.
38
+template<typename scalar_t>
39
+__global__ void fused_add_rms_norm_kernel(
40
+ scalar_t* __restrict__ input, // [..., hidden_size]
41
+ scalar_t* __restrict__ residual, // [..., hidden_size]
42
+ const scalar_t* __restrict__ weight, // [hidden_size]
43
+ const float epsilon,
44
+ const int num_tokens,
45
+ const int hidden_size) {
46
+ __shared__ float s_variance;
47
+ float variance = 0.0f;
48
+
49
+ for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
50
+ float x = (float) input[blockIdx.x * hidden_size + idx];
51
+ x += (float) residual[blockIdx.x * hidden_size + idx];
52
+ variance += x * x;
53
+ residual[blockIdx.x * hidden_size + idx] = (scalar_t) x;
54
+ }
55
+ variance = blockReduceSum<float>(variance);
56
+ if (threadIdx.x == 0) {
57
+ s_variance = rsqrtf(variance / hidden_size + epsilon);
58
+ }
59
+ __syncthreads();
60
+
61
+ for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
62
+ float x = (float) residual[blockIdx.x * hidden_size + idx];
63
+ input[blockIdx.x * hidden_size + idx] = ((scalar_t) (x * s_variance)) * weight[idx];
64
+ }
65
+}
66
+
67
} // namespace vllm
68
69
void rms_norm(
@@ -60,3 +90,28 @@ void rms_norm(
90
hidden_size);
91
});
92
}
93
+
94
+void fused_add_rms_norm(
95
+ torch::Tensor& input, // [..., hidden_size]
96
+ torch::Tensor& residual, // [..., hidden_size]
97
+ torch::Tensor& weight, // [hidden_size]
98
+ float epsilon) {
99
+ int hidden_size = input.size(-1);
100
+ int num_tokens = input.numel() / hidden_size;
101
+
102
+ dim3 grid(num_tokens);
103
+ dim3 block(std::min(hidden_size, 1024));
104
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
105
+ VLLM_DISPATCH_FLOATING_TYPES(
106
+ input.scalar_type(),
107
+ "fused_add_rms_norm_kernel",
108
+ [&] {
109
+ vllm::fused_add_rms_norm_kernel<scalar_t><<<grid, block, 0, stream>>>(
110
+ input.data_ptr<scalar_t>(),
111
+ residual.data_ptr<scalar_t>(),
112
+ weight.data_ptr<scalar_t>(),
113
+ epsilon,
114
+ num_tokens,
115
+ hidden_size);
116
+ });
117
+}
vllm/model_executor/layers/layernorm.py
+15
-1
@@ -1,4 +1,6 @@
1
"""Custom normalization layers."""
2
+from typing import Optional, Tuple, Union
3
+
4
import torch
5
import torch.nn as nn
6
@@ -21,7 +23,19 @@ class RMSNorm(nn.Module):
23
self.weight = nn.Parameter(torch.ones(hidden_size))
24
self.variance_epsilon = eps
25
24
- def forward(self, x: torch.Tensor) -> torch.Tensor:
26
+ def forward(
27
+ self,
28
+ x: torch.Tensor,
29
+ residual: Optional[torch.Tensor] = None,
30
+ ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
31
+ if residual is not None:
32
+ layernorm_ops.fused_add_rms_norm(
33
+ x,
34
+ residual,
35
+ self.weight.data,
36
+ self.variance_epsilon,
37
+ )
38
+ return x, residual
39
out = torch.empty_like(x)
40
layernorm_ops.rms_norm(
41
out,
vllm/model_executor/models/baichuan.py
+15
-10
@@ -225,10 +225,15 @@ class BaiChuanDecoderLayer(nn.Module):
225
kv_cache: KVCache,
226
input_metadata: InputMetadata,
227
cache_event: Optional[torch.cuda.Event],
228
- ) -> torch.Tensor:
228
+ residual: Optional[torch.Tensor],
229
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
230
# Self Attention
230
- residual = hidden_states
231
- hidden_states = self.input_layernorm(hidden_states)
231
+ if residual is None:
232
+ residual = hidden_states
233
+ hidden_states = self.input_layernorm(hidden_states)
234
+ else:
235
+ hidden_states, residual = self.input_layernorm(
236
+ hidden_states, residual)
237
hidden_states = self.self_attn(
238
positions=positions,
239
hidden_states=hidden_states,
@@ -236,14 +241,12 @@ class BaiChuanDecoderLayer(nn.Module):
241
input_metadata=input_metadata,
242
cache_event=cache_event,
243
)
239
- hidden_states = residual + hidden_states
244
245
# Fully Connected
242
- residual = hidden_states
243
- hidden_states = self.post_attention_layernorm(hidden_states)
246
+ hidden_states, residual = self.post_attention_layernorm(
247
+ hidden_states, residual)
248
hidden_states = self.mlp(hidden_states)
245
- hidden_states = residual + hidden_states
246
- return hidden_states
249
+ return hidden_states, residual
250
251
252
class BaiChuanModel(nn.Module):
@@ -276,20 +279,22 @@ class BaiChuanModel(nn.Module):
279
cache_events: Optional[List[torch.cuda.Event]],
280
) -> torch.Tensor:
281
hidden_states = self.embed_tokens(input_ids)
282
+ residual = None
283
for i in range(len(self.layers)):
284
if cache_events is None:
285
cache_event = None
286
else:
287
cache_event = cache_events[i]
288
layer = self.layers[i]
285
- hidden_states = layer(
289
+ hidden_states, residual = layer(
290
positions,
291
hidden_states,
292
kv_caches[i],
293
input_metadata,
294
cache_event,
295
+ residual,
296
)
292
- hidden_states = self.norm(hidden_states)
297
+ hidden_states, _ = self.norm(hidden_states, residual)
298
return hidden_states
299
300
vllm/model_executor/models/internlm.py
+15
-10
@@ -155,10 +155,15 @@ class InternLMDecoderLayer(nn.Module):
155
kv_cache: KVCache,
156
input_metadata: InputMetadata,
157
cache_event: Optional[torch.cuda.Event],
158
- ) -> torch.Tensor:
158
+ residual: Optional[torch.Tensor],
159
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
160
# Self Attention
160
- residual = hidden_states
161
- hidden_states = self.input_layernorm(hidden_states)
161
+ if residual is None:
162
+ residual = hidden_states
163
+ hidden_states = self.input_layernorm(hidden_states)
164
+ else:
165
+ hidden_states, residual = self.input_layernorm(
166
+ hidden_states, residual)
167
hidden_states = self.self_attn(
168
positions=positions,
169
hidden_states=hidden_states,
@@ -166,14 +171,12 @@ class InternLMDecoderLayer(nn.Module):
171
input_metadata=input_metadata,
172
cache_event=cache_event,
173
)
169
- hidden_states = residual + hidden_states
174
175
# Fully Connected
172
- residual = hidden_states
173
- hidden_states = self.post_attention_layernorm(hidden_states)
176
+ hidden_states, residual = self.post_attention_layernorm(
177
+ hidden_states, residual)
178
hidden_states = self.mlp(hidden_states)
175
- hidden_states = residual + hidden_states
176
- return hidden_states
179
+ return hidden_states, residual
180
181
182
class InternLMModel(nn.Module):
@@ -208,20 +211,22 @@ class InternLMModel(nn.Module):
211
cache_events: Optional[List[torch.cuda.Event]],
212
) -> torch.Tensor:
213
hidden_states = self.embed_tokens(input_ids)
214
+ residual = None
215
for i in range(len(self.layers)):
216
if cache_events is None:
217
cache_event = None
218
else:
219
cache_event = cache_events[i]
220
layer = self.layers[i]
217
- hidden_states = layer(
221
+ hidden_states, residual = layer(
222
positions,
223
hidden_states,
224
kv_caches[i],
225
input_metadata,
226
cache_event,
227
+ residual,
228
)
224
- hidden_states = self.norm(hidden_states)
229
+ hidden_states, _ = self.norm(hidden_states, residual)
230
return hidden_states
231
232
vllm/model_executor/models/llama.py
+15
-10
@@ -197,10 +197,15 @@ class LlamaDecoderLayer(nn.Module):
197
kv_cache: KVCache,
198
input_metadata: InputMetadata,
199
cache_event: Optional[torch.cuda.Event],
200
- ) -> torch.Tensor:
200
+ residual: Optional[torch.Tensor],
201
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
202
# Self Attention
202
- residual = hidden_states
203
- hidden_states = self.input_layernorm(hidden_states)
203
+ if residual is None:
204
+ residual = hidden_states
205
+ hidden_states = self.input_layernorm(hidden_states)
206
+ else:
207
+ hidden_states, residual = self.input_layernorm(
208
+ hidden_states, residual)
209
hidden_states = self.self_attn(
210
positions=positions,
211
hidden_states=hidden_states,
@@ -208,14 +213,12 @@ class LlamaDecoderLayer(nn.Module):
213
input_metadata=input_metadata,
214
cache_event=cache_event,
215
)
211
- hidden_states = residual + hidden_states
216
217
# Fully Connected
214
- residual = hidden_states
215
- hidden_states = self.post_attention_layernorm(hidden_states)
218
+ hidden_states, residual = self.post_attention_layernorm(
219
+ hidden_states, residual)
220
hidden_states = self.mlp(hidden_states)
217
- hidden_states = residual + hidden_states
218
- return hidden_states
221
+ return hidden_states, residual
222
223
224
class LlamaModel(nn.Module):
@@ -248,20 +251,22 @@ class LlamaModel(nn.Module):
251
cache_events: Optional[List[torch.cuda.Event]],
252
) -> torch.Tensor:
253
hidden_states = self.embed_tokens(input_ids)
254
+ residual = None
255
for i in range(len(self.layers)):
256
if cache_events is None:
257
cache_event = None
258
else:
259
cache_event = cache_events[i]
260
layer = self.layers[i]
257
- hidden_states = layer(
261
+ hidden_states, residual = layer(
262
positions,
263
hidden_states,
264
kv_caches[i],
265
input_metadata,
266
cache_event,
267
+ residual,
268
)
264
- hidden_states = self.norm(hidden_states)
269
+ hidden_states, _ = self.norm(hidden_states, residual)
270
return hidden_states
271
272
vllm/model_executor/models/mistral.py
+15
-10
@@ -191,10 +191,15 @@ class MistralDecoderLayer(nn.Module):
191
kv_cache: KVCache,
192
input_metadata: InputMetadata,
193
cache_event: Optional[torch.cuda.Event],
194
- ) -> torch.Tensor:
194
+ residual: Optional[torch.Tensor],
195
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
196
# Self Attention
196
- residual = hidden_states
197
- hidden_states = self.input_layernorm(hidden_states)
197
+ if residual is None:
198
+ residual = hidden_states
199
+ hidden_states = self.input_layernorm(hidden_states)
200
+ else:
201
+ hidden_states, residual = self.input_layernorm(
202
+ hidden_states, residual)
203
hidden_states = self.self_attn(
204
positions=positions,
205
hidden_states=hidden_states,
@@ -202,14 +207,12 @@ class MistralDecoderLayer(nn.Module):
207
input_metadata=input_metadata,
208
cache_event=cache_event,
209
)
205
- hidden_states = residual + hidden_states
210
211
# Fully Connected
208
- residual = hidden_states
209
- hidden_states = self.post_attention_layernorm(hidden_states)
212
+ hidden_states, residual = self.post_attention_layernorm(
213
+ hidden_states, residual)
214
hidden_states = self.mlp(hidden_states)
211
- hidden_states = residual + hidden_states
212
- return hidden_states
215
+ return hidden_states, residual
216
217
218
class MistralModel(nn.Module):
@@ -243,20 +246,22 @@ class MistralModel(nn.Module):
246
cache_events: Optional[List[torch.cuda.Event]],
247
) -> torch.Tensor:
248
hidden_states = self.embed_tokens(input_ids)
249
+ residual = None
250
for i in range(len(self.layers)):
251
if cache_events is None:
252
cache_event = None
253
else:
254
cache_event = cache_events[i]
255
layer = self.layers[i]
252
- hidden_states = layer(
256
+ hidden_states, residual = layer(
257
positions,
258
hidden_states,
259
kv_caches[i],
260
input_metadata,
261
cache_event,
262
+ residual,
263
)
259
- hidden_states = self.norm(hidden_states)
264
+ hidden_states, _ = self.norm(hidden_states, residual)
265
return hidden_states
266
267
vllm/model_executor/models/qwen.py
+13
-10
@@ -159,10 +159,14 @@ class QWenBlock(nn.Module):
159
kv_cache: KVCache,
160
input_metadata: InputMetadata,
161
cache_event: Optional[torch.cuda.Event],
162
- ) -> torch.Tensor:
162
+ residual: Optional[torch.Tensor],
163
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
164
# Self Attention
164
- residual = hidden_states
165
- hidden_states = self.ln_1(hidden_states)
165
+ if residual is None:
166
+ residual = hidden_states
167
+ hidden_states = self.ln_1(hidden_states)
168
+ else:
169
+ hidden_states, residual = self.ln_1(hidden_states, residual)
170
hidden_states = self.attn(
171
positions=positions,
172
hidden_states=hidden_states,
@@ -170,14 +174,11 @@ class QWenBlock(nn.Module):
174
input_metadata=input_metadata,
175
cache_event=cache_event,
176
)
173
- hidden_states = residual + hidden_states
177
178
# Fully Connected
176
- residual = hidden_states
177
- hidden_states = self.ln_2(hidden_states)
179
+ hidden_states, residual = self.ln_2(hidden_states, residual)
180
hidden_states = self.mlp(hidden_states)
179
- hidden_states = residual + hidden_states
180
- return hidden_states
181
+ return hidden_states, residual
182
183
184
class QWenModel(nn.Module):
@@ -210,20 +211,22 @@ class QWenModel(nn.Module):
211
cache_events: Optional[List[torch.cuda.Event]],
212
) -> torch.Tensor:
213
hidden_states = self.wte(input_ids)
214
+ residual = None
215
for i in range(len(self.h)):
216
if cache_events is None:
217
cache_event = None
218
else:
219
cache_event = cache_events[i]
220
layer = self.h[i]
219
- hidden_states = layer(
221
+ hidden_states, residual = layer(
222
positions,
223
hidden_states,
224
kv_caches[i],
225
input_metadata,
226
cache_event,
227
+ residual,
228
)
226
- hidden_states = self.ln_f(hidden_states)
229
+ hidden_states, _ = self.ln_f(hidden_states, residual)
230
return hidden_states
231
232
vllm/model_executor/models/yi.py
+13
-10
@@ -195,10 +195,14 @@ class YiDecoderLayer(nn.Module):
195
kv_cache: KVCache,
196
input_metadata: InputMetadata,
197
cache_event: Optional[torch.cuda.Event],
198
- ) -> torch.Tensor:
198
+ residual: Optional[torch.Tensor],
199
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
200
# Self Attention
200
- residual = hidden_states
201
- hidden_states = self.ln1(hidden_states)
201
+ if residual is None:
202
+ residual = hidden_states
203
+ hidden_states = self.ln1(hidden_states)
204
+ else:
205
+ hidden_states, residual = self.ln1(hidden_states, residual)
206
hidden_states = self.self_attn(
207
positions=positions,
208
hidden_states=hidden_states,
@@ -206,14 +210,11 @@ class YiDecoderLayer(nn.Module):
210
input_metadata=input_metadata,
211
cache_event=cache_event,
212
)
209
- hidden_states = residual + hidden_states
213
214
# Fully Connected
212
- residual = hidden_states
213
- hidden_states = self.ln2(hidden_states)
215
+ hidden_states, residual = self.ln2(hidden_states, residual)
216
hidden_states = self.mlp(hidden_states)
215
- hidden_states = residual + hidden_states
216
- return hidden_states
217
+ return hidden_states, residual
218
219
220
class YiModel(nn.Module):
@@ -246,20 +247,22 @@ class YiModel(nn.Module):
247
cache_events: Optional[List[torch.cuda.Event]],
248
) -> torch.Tensor:
249
hidden_states = self.embed_tokens(input_ids)
250
+ residual = None
251
for i in range(len(self.layers)):
252
if cache_events is None:
253
cache_event = None
254
else:
255
cache_event = cache_events[i]
256
layer = self.layers[i]
255
- hidden_states = layer(
257
+ hidden_states, residual = layer(
258
positions,
259
hidden_states,
260
kv_caches[i],
261
input_metadata,
262
cache_event,
263
+ residual,
264
)
262
- hidden_states = self.norm(hidden_states)
265
+ hidden_states, _ = self.norm(hidden_states, residual)
266
return hidden_states
267
268