A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Kernel] Layernorm performance optimization (#3662)
mawong-amd committed
Mar 30, 2024 at 14:26 UTC
b6d103542c654fb63013a1e45a586d654ae36a2a
4 files changed
+285
-47
cmake/utils.cmake
+5
@@ -100,6 +100,11 @@ function (get_torch_gpu_compiler_flags OUT_GPU_FLAGS GPU_LANG)
100
101
if (CUDA_VERSION VERSION_GREATER_EQUAL 11.8)
102
list(APPEND GPU_FLAGS "-DENABLE_FP8_E5M2")
103
+ list(REMOVE_ITEM GPU_FLAGS
104
+ "-D__CUDA_NO_HALF_OPERATORS__"
105
+ "-D__CUDA_NO_HALF_CONVERSIONS__"
106
+ "-D__CUDA_NO_BFLOAT16_CONVERSIONS__"
107
+ "-D__CUDA_NO_HALF2_OPERATORS__")
108
endif()
109
110
elseif(${GPU_LANG} STREQUAL "HIP")
csrc/layernorm_kernels.cu
+250
-20
@@ -4,6 +4,16 @@
4
5
#include "dispatch_utils.h"
6
#include "reduction_utils.cuh"
7
+#ifndef USE_ROCM
8
+ #include <cuda_bf16.h>
9
+ #include <cuda_fp16.h>
10
+#else
11
+ #include <hip/hip_bf16.h>
12
+ #include <hip/hip_fp16.h>
13
+
14
+ using __nv_bfloat16 = __hip_bfloat16;
15
+ using __nv_bfloat162 = __hip_bfloat162;
16
+#endif
17
18
namespace vllm {
19
@@ -35,9 +45,199 @@ __global__ void rms_norm_kernel(
45
}
46
}
47
38
-// TODO: Further optimize this kernel.
39
-template<typename scalar_t>
40
-__global__ void fused_add_rms_norm_kernel(
48
+
49
+/* Converter structs for the conversion from torch types to HIP/CUDA types,
50
+ and the associated type conversions within HIP/CUDA. These helpers need
51
+ to be implemented for now because the relevant type conversion
52
+ operators/constructors are not consistently implemented by HIP/CUDA, so
53
+ a generic conversion via type casts cannot be implemented.
54
+
55
+ Each struct should have the member static constexpr bool `exists`:
56
+ If false, the optimized kernel is not used for the corresponding torch type.
57
+ If true, the struct should be fully defined as shown in the examples below.
58
+ */
59
+template<typename torch_type>
60
+struct _typeConvert { static constexpr bool exists = false; };
61
+
62
+template<>
63
+struct _typeConvert<c10::Half> {
64
+ static constexpr bool exists = true;
65
+ using hip_type = __half;
66
+ using packed_hip_type = __half2;
67
+
68
+ __device__ static inline float convert(hip_type x) { return __half2float(x); }
69
+ __device__ static inline float2 convert(packed_hip_type x) { return __half22float2(x); }
70
+ __device__ static inline hip_type convert(float x) { return __float2half_rn(x); }
71
+ __device__ static inline packed_hip_type convert(float2 x) { return __float22half2_rn(x); }
72
+};
73
+
74
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
75
+// CUDA_ARCH < 800 does not have BF16 support
76
+// TODO: Add in ROCm support once public headers handle bf16 maturely
77
+template<>
78
+struct _typeConvert<c10::BFloat16> {
79
+ static constexpr bool exists = true;
80
+ using hip_type = __nv_bfloat16;
81
+ using packed_hip_type = __nv_bfloat162;
82
+
83
+ __device__ static inline float convert(hip_type x) { return __bfloat162float(x); }
84
+ __device__ static inline float2 convert(packed_hip_type x) { return __bfloat1622float2(x); }
85
+ __device__ static inline hip_type convert(float x) { return __float2bfloat16(x); }
86
+ __device__ static inline packed_hip_type convert(float2 x) { return __float22bfloat162_rn(x); }
87
+};
88
+#endif
89
+
90
+
91
+/* Vector POD struct to generate vectorized and packed FP16/BF16 ops
92
+ for appropriate specializations of fused_add_rms_norm_kernel.
93
+ Only functions that are necessary in that kernel are implemented.
94
+ Alignment to 16 bytes is required to use 128-bit global memory ops.
95
+ */
96
+template<typename scalar_t, int width>
97
+struct alignas(16) _f16Vec {
98
+ /* Not theoretically necessary that width is a power of 2 but should
99
+ almost always be the case for optimization purposes */
100
+ static_assert(width > 0 && (width & (width - 1)) == 0,
101
+ "Width is not a positive power of 2!");
102
+ using Converter = _typeConvert<scalar_t>;
103
+ using T1 = typename Converter::hip_type;
104
+ using T2 = typename Converter::packed_hip_type;
105
+ T1 data[width];
106
+
107
+ __device__ _f16Vec& operator+=(const _f16Vec<scalar_t, width>& other) {
108
+ if constexpr (width % 2 == 0) {
109
+ #pragma unroll
110
+ for (int i = 0; i < width; i += 2) {
111
+ T2 temp{data[i], data[i+1]};
112
+ temp += T2{other.data[i], other.data[i+1]};
113
+ data[i] = temp.x;
114
+ data[i+1] = temp.y;
115
+ }
116
+ } else {
117
+ #pragma unroll
118
+ for (int i = 0; i < width; ++i)
119
+ data[i] += other.data[i];
120
+ }
121
+ return *this;
122
+ }
123
+
124
+ __device__ _f16Vec& operator*=(const _f16Vec<scalar_t, width>& other) {
125
+ if constexpr (width % 2 == 0) {
126
+ #pragma unroll
127
+ for (int i = 0; i < width; i += 2) {
128
+ T2 temp{data[i], data[i+1]};
129
+ temp *= T2{other.data[i], other.data[i+1]};
130
+ data[i] = temp.x;
131
+ data[i+1] = temp.y;
132
+ }
133
+ } else {
134
+ #pragma unroll
135
+ for (int i = 0; i < width; ++i)
136
+ data[i] *= other.data[i];
137
+ }
138
+ return *this;
139
+ }
140
+
141
+ __device__ _f16Vec& operator*=(const float scale) {
142
+ if constexpr (width % 2 == 0) {
143
+ #pragma unroll
144
+ for (int i = 0; i < width; i += 2) {
145
+ float2 temp_f = Converter::convert(T2{data[i], data[i+1]});
146
+ temp_f.x *= scale;
147
+ temp_f.y *= scale;
148
+ T2 temp = Converter::convert(temp_f);
149
+ data[i] = temp.x;
150
+ data[i+1] = temp.y;
151
+ }
152
+ } else {
153
+ #pragma unroll
154
+ for (int i = 0; i < width; ++i) {
155
+ float temp = Converter::convert(data[i]) * scale;
156
+ data[i] = Converter::convert(temp);
157
+ }
158
+ }
159
+ return *this;
160
+ }
161
+
162
+ __device__ float sum_squares() const {
163
+ float result = 0.0f;
164
+ if constexpr (width % 2 == 0) {
165
+ #pragma unroll
166
+ for (int i = 0; i < width; i += 2) {
167
+ float2 z = Converter::convert(T2{data[i], data[i+1]});
168
+ result += z.x * z.x + z.y * z.y;
169
+ }
170
+ } else {
171
+ #pragma unroll
172
+ for (int i = 0; i < width; ++i) {
173
+ float x = Converter::convert(data[i]);
174
+ result += x * x;
175
+ }
176
+ }
177
+ return result;
178
+ }
179
+};
180
+
181
+/* Function specialization in the case of FP16/BF16 tensors.
182
+ Additional optimizations we can make in this case are
183
+ packed and vectorized operations, which help with the
184
+ memory latency bottleneck. */
185
+template<typename scalar_t, int width>
186
+__global__ std::enable_if_t<
187
+ (width > 0) && _typeConvert<scalar_t>::exists> fused_add_rms_norm_kernel(
188
+ scalar_t* __restrict__ input, // [..., hidden_size]
189
+ scalar_t* __restrict__ residual, // [..., hidden_size]
190
+ const scalar_t* __restrict__ weight, // [hidden_size]
191
+ const float epsilon,
192
+ const int num_tokens,
193
+ const int hidden_size) {
194
+ // Sanity checks on our vector struct and type-punned pointer arithmetic
195
+ static_assert(std::is_pod_v<_f16Vec<scalar_t, width>>);
196
+ static_assert(sizeof(_f16Vec<scalar_t, width>) == sizeof(scalar_t) * width);
197
+
198
+ const int vec_hidden_size = hidden_size / width;
199
+ __shared__ float s_variance;
200
+ float variance = 0.0f;
201
+ /* These and the argument pointers are all declared `restrict` as they are
202
+ not aliased in practice. Argument pointers should not be dereferenced
203
+ in this kernel as that would be undefined behavior */
204
+ auto* __restrict__ input_v = reinterpret_cast<_f16Vec<scalar_t, width>*>(input);
205
+ auto* __restrict__ residual_v = reinterpret_cast<_f16Vec<scalar_t, width>*>(residual);
206
+ auto* __restrict__ weight_v = reinterpret_cast<const _f16Vec<scalar_t, width>*>(weight);
207
+
208
+ for (int idx = threadIdx.x; idx < vec_hidden_size; idx += blockDim.x) {
209
+ int id = blockIdx.x * vec_hidden_size + idx;
210
+ _f16Vec<scalar_t, width> temp = input_v[id];
211
+ temp += residual_v[id];
212
+ variance += temp.sum_squares();
213
+ residual_v[id] = temp;
214
+ }
215
+ /* Keep the following if-else block in sync with the
216
+ calculation of max_block_size in fused_add_rms_norm */
217
+ if (num_tokens < 256) {
218
+ variance = blockReduceSum<float, 1024>(variance);
219
+ } else variance = blockReduceSum<float, 256>(variance);
220
+ if (threadIdx.x == 0) {
221
+ s_variance = rsqrtf(variance / hidden_size + epsilon);
222
+ }
223
+ __syncthreads();
224
+
225
+ for (int idx = threadIdx.x; idx < vec_hidden_size; idx += blockDim.x) {
226
+ int id = blockIdx.x * vec_hidden_size + idx;
227
+ _f16Vec<scalar_t, width> temp = residual_v[id];
228
+ temp *= s_variance;
229
+ temp *= weight_v[idx];
230
+ input_v[id] = temp;
231
+ }
232
+}
233
+
234
+
235
+/* Generic fused_add_rms_norm_kernel
236
+ The width field is not used here but necessary for other specializations.
237
+ */
238
+template<typename scalar_t, int width>
239
+__global__ std::enable_if_t<
240
+ (width == 0) || !_typeConvert<scalar_t>::exists> fused_add_rms_norm_kernel(
241
scalar_t* __restrict__ input, // [..., hidden_size]
242
scalar_t* __restrict__ residual, // [..., hidden_size]
243
const scalar_t* __restrict__ weight, // [hidden_size]
@@ -48,12 +248,17 @@ __global__ void fused_add_rms_norm_kernel(
248
float variance = 0.0f;
249
250
for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
51
- float x = (float) input[blockIdx.x * hidden_size + idx];
52
- x += (float) residual[blockIdx.x * hidden_size + idx];
251
+ scalar_t z = input[blockIdx.x * hidden_size + idx];
252
+ z += residual[blockIdx.x * hidden_size + idx];
253
+ float x = (float) z;
254
variance += x * x;
54
- residual[blockIdx.x * hidden_size + idx] = (scalar_t) x;
255
+ residual[blockIdx.x * hidden_size + idx] = z;
256
}
56
- variance = blockReduceSum<float>(variance);
257
+ /* Keep the following if-else block in sync with the
258
+ calculation of max_block_size in fused_add_rms_norm */
259
+ if (num_tokens < 256) {
260
+ variance = blockReduceSum<float, 1024>(variance);
261
+ } else variance = blockReduceSum<float, 256>(variance);
262
if (threadIdx.x == 0) {
263
s_variance = rsqrtf(variance / hidden_size + epsilon);
264
}
@@ -93,6 +298,21 @@ void rms_norm(
298
});
299
}
300
301
+#define LAUNCH_FUSED_ADD_RMS_NORM(width) \
302
+ VLLM_DISPATCH_FLOATING_TYPES( \
303
+ input.scalar_type(), \
304
+ "fused_add_rms_norm_kernel", \
305
+ [&] { \
306
+ vllm::fused_add_rms_norm_kernel \
307
+ <scalar_t, width><<<grid, block, 0, stream>>>( \
308
+ input.data_ptr<scalar_t>(), \
309
+ residual.data_ptr<scalar_t>(), \
310
+ weight.data_ptr<scalar_t>(), \
311
+ epsilon, \
312
+ num_tokens, \
313
+ hidden_size); \
314
+ });
315
+
316
void fused_add_rms_norm(
317
torch::Tensor& input, // [..., hidden_size]
318
torch::Tensor& residual, // [..., hidden_size]
@@ -102,19 +322,29 @@ void fused_add_rms_norm(
322
int num_tokens = input.numel() / hidden_size;
323
324
dim3 grid(num_tokens);
105
- dim3 block(std::min(hidden_size, 1024));
325
+ /* This kernel is memory-latency bound in many scenarios.
326
+ When num_tokens is large, a smaller block size allows
327
+ for increased block occupancy on CUs and better latency
328
+ hiding on global mem ops. */
329
+ const int max_block_size = (num_tokens < 256) ? 1024 : 256;
330
+ dim3 block(std::min(hidden_size, max_block_size));
331
const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
332
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
108
- VLLM_DISPATCH_FLOATING_TYPES(
109
- input.scalar_type(),
110
- "fused_add_rms_norm_kernel",
111
- [&] {
112
- vllm::fused_add_rms_norm_kernel<scalar_t><<<grid, block, 0, stream>>>(
113
- input.data_ptr<scalar_t>(),
114
- residual.data_ptr<scalar_t>(),
115
- weight.data_ptr<scalar_t>(),
116
- epsilon,
117
- num_tokens,
118
- hidden_size);
119
- });
333
+ /*If the tensor types are FP16/BF16, try to use the optimized kernel
334
+ with packed + vectorized ops.
335
+ Max optimization is achieved with a width-8 vector of FP16/BF16s
336
+ since we can load at most 128 bits at once in a global memory op.
337
+ However, this requires each tensor's data to be aligned to 16
338
+ bytes.
339
+ */
340
+ auto inp_ptr = reinterpret_cast<std::uintptr_t>(input.data_ptr());
341
+ auto res_ptr = reinterpret_cast<std::uintptr_t>(residual.data_ptr());
342
+ auto wt_ptr = reinterpret_cast<std::uintptr_t>(weight.data_ptr());
343
+ bool ptrs_are_aligned = inp_ptr % 16 == 0 && res_ptr % 16 == 0 \
344
+ && wt_ptr % 16 == 0;
345
+ if (ptrs_are_aligned && hidden_size % 8 == 0) {
346
+ LAUNCH_FUSED_ADD_RMS_NORM(8);
347
+ } else {
348
+ LAUNCH_FUSED_ADD_RMS_NORM(0);
349
+ }
350
}
csrc/reduction_utils.cuh
+28
-26
@@ -20,43 +20,45 @@
20
#include "cuda_compat.h"
21
22
namespace vllm {
23
-
24
-template<typename T>
23
+template<typename T, int numLanes = WARP_SIZE>
24
__inline__ __device__ T warpReduceSum(T val) {
26
-#pragma unroll
27
- for (int mask = WARP_SIZE/2; mask > 0; mask >>= 1)
25
+ static_assert(numLanes > 0 && (numLanes & (numLanes - 1)) == 0,
26
+ "numLanes is not a positive power of 2!");
27
+ static_assert(numLanes <= WARP_SIZE);
28
+ #pragma unroll
29
+ for (int mask = numLanes >> 1; mask > 0; mask >>= 1)
30
val += VLLM_SHFL_XOR_SYNC(val, mask);
31
return val;
32
}
33
32
-__inline__ __device__ constexpr int _calculateLaneMask(int warp_size) {
33
- return warp_size - 1;
34
-}
35
-
36
-__inline__ __device__ constexpr int _calculateWidShift(int warp_size) {
37
- return 5 + (warp_size >> 6);
34
+// Helper function to return the next largest power of 2
35
+static constexpr int _nextPow2(unsigned int num) {
36
+ if (num <= 1) return num;
37
+ return 1 << (CHAR_BIT * sizeof(num) - __builtin_clz(num - 1));
38
}
39
40
/* Calculate the sum of all elements in a block */
41
-template<typename T>
41
+template<typename T, int maxBlockSize = 1024>
42
__inline__ __device__ T blockReduceSum(T val) {
43
- static __shared__ T shared[WARP_SIZE];
44
- constexpr auto LANE_MASK = _calculateLaneMask(WARP_SIZE);
45
- constexpr auto WID_SHIFT = _calculateWidShift(WARP_SIZE);
46
- int lane = threadIdx.x & LANE_MASK;
47
- int wid = threadIdx.x >> WID_SHIFT;
48
-
49
- val = warpReduceSum<T>(val);
50
-
51
- if (lane == 0)
52
- shared[wid] = val;
43
+ static_assert(maxBlockSize <= 1024);
44
+ if constexpr (maxBlockSize > WARP_SIZE) {
45
+ val = warpReduceSum<T>(val);
46
+ // Calculates max number of lanes that need to participate in the last warpReduce
47
+ constexpr int maxActiveLanes = (maxBlockSize + WARP_SIZE - 1) / WARP_SIZE;
48
+ static __shared__ T shared[maxActiveLanes];
49
+ int lane = threadIdx.x % WARP_SIZE;
50
+ int wid = threadIdx.x / WARP_SIZE;
51
+ if (lane == 0)
52
+ shared[wid] = val;
53
54
- __syncthreads();
54
+ __syncthreads();
55
56
- // Modify from blockDim.x << 5 to blockDim.x / 32. to prevent
57
- // blockDim.x is not divided by 32
58
- val = (threadIdx.x < (blockDim.x / (WARP_SIZE * 1.0f))) ? shared[lane] : (T)(0.0f);
59
- val = warpReduceSum<T>(val);
56
+ val = (threadIdx.x < blockDim.x / float(WARP_SIZE)) ? shared[lane] : (T)(0.0f);
57
+ val = warpReduceSum<T, _nextPow2(maxActiveLanes)>(val);
58
+ } else {
59
+ // A single warpReduce is equal to blockReduce
60
+ val = warpReduceSum<T, _nextPow2(maxBlockSize)>(val);
61
+ }
62
return val;
63
}
64
tests/kernels/test_layernorm.py
+2
-1
@@ -5,7 +5,8 @@ from vllm.model_executor.layers.layernorm import RMSNorm
5
6
DTYPES = [torch.half, torch.bfloat16, torch.float]
7
NUM_TOKENS = [7, 83, 4096] # Arbitrary values for testing
8
-HIDDEN_SIZES = [768, 5120, 8192] # Arbitrary values for testing
8
+HIDDEN_SIZES = [768, 769, 770, 771, 5120, 5124, 5125, 5126, 8192,
9
+ 8199] # Arbitrary values for testing
10
ADD_RESIDUAL = [False, True]
11
SEEDS = [0]
12
CUDA_DEVICES = [