@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 09e9245478
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add custom kernel for RMS normalization (#16)

Woosuk Kwon committed Mar 31, 2023 at 09:51 UTC 09e9245478a44faec3c9bc888edea4089085e222
9 files changed +243 -58
cacheflow/models/layernorm.py new
+26
@@ -0,0 +1,26 @@
1 +import torch
2 +import torch.nn as nn
3 +
4 +from cacheflow import layernorm_ops
5 +
6 +
7 +class RMSNorm(nn.Module):
8 +
9 + def __init__(
10 + self,
11 + hidden_size: int,
12 + eps: float = 1e-6,
13 + ) -> None:
14 + super().__init__()
15 + self.weight = nn.Parameter(torch.ones(hidden_size))
16 + self.variance_epsilon = eps
17 +
18 + def forward(self, x: torch.Tensor) -> torch.Tensor:
19 + out = torch.empty_like(x)
20 + layernorm_ops.rms_norm(
21 + out,
22 + x,
23 + self.weight.data,
24 + self.variance_epsilon,
25 + )
26 + return out
cacheflow/models/llama.py
+4 -19
@@ -12,6 +12,7 @@ from transformers import LlamaConfig
12
13 from cacheflow.models import InputMetadata
14 from cacheflow.models.attention import LlamaCacheFlowAttention
15 +from cacheflow.models.layernorm import RMSNorm
16 from cacheflow.models.sample import Sampler
17 from cacheflow.parallel_utils.parallel_state import (
18 get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
@@ -23,22 +24,6 @@ from cacheflow.sequence import SequenceOutputs
24 KVCache = Tuple[torch.Tensor, torch.Tensor]
25
26
26 -class LlamaRMSNorm(nn.Module):
27 -
28 - def __init__(self, hidden_size, eps=1e-6):
29 - super().__init__()
30 - self.weight = nn.Parameter(torch.ones(hidden_size))
31 - self.variance_epsilon = eps
32 -
33 - def forward(self, hidden_states):
34 - variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
35 - hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
36 - # convert into half-precision if necessary
37 - if self.weight.dtype in [torch.float16, torch.bfloat16]:
38 - hidden_states = hidden_states.to(self.weight.dtype)
39 - return self.weight * hidden_states
40 -
41 -
27 class LlamaMLP(nn.Module):
28
29 def __init__(
@@ -148,8 +133,8 @@ class LlamaDecoderLayer(nn.Module):
133 intermediate_size=config.intermediate_size,
134 hidden_act=config.hidden_act,
135 )
151 - self.input_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
152 - self.post_attention_layernorm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
136 + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
137 + self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
138
139 def forward(
140 self,
@@ -190,7 +175,7 @@ class LlamaModel(nn.Module):
175 self.embed_tokens = VocabParallelEmbedding(config.vocab_size, config.hidden_size,
176 perform_initialization=False)
177 self.layers = nn.ModuleList([LlamaDecoderLayer(config) for _ in range(config.num_hidden_layers)])
193 - self.norm = LlamaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
178 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
179
180 def forward(
181 self,
csrc/attention_kernels.cu
+1
@@ -3,6 +3,7 @@
3
4 #include "attention_utils.h"
5 #include "cuda_primitives.h"
6 +#include "reduction_utils.h"
7
8 #include <algorithm>
9
csrc/attention_utils.h
-39
@@ -159,45 +159,6 @@ struct Qk_dot<uint16_t, 4> {
159 }
160 };
161
162 -////////////////////////////////////////////////////////////////////////////////////////////////////
163 -
164 -template<int WARPS_PER_BLOCK, int WARP_SIZE = 32>
165 -inline __device__ float block_sum(float* red_smem, float sum)
166 -{
167 -
168 - // Decompose the thread index into warp / lane.
169 - int warp = threadIdx.x / WARP_SIZE;
170 - int lane = threadIdx.x % WARP_SIZE;
171 -
172 - // Compute the sum per warp.
173 -#pragma unroll
174 - for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
175 - sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
176 - }
177 -
178 - // Warp leaders store the data to shared memory.
179 - if (lane == 0) {
180 - red_smem[warp] = sum;
181 - }
182 -
183 - // Make sure the data is in shared memory.
184 - __syncthreads();
185 -
186 - // The warps compute the final sums.
187 - if (lane < WARPS_PER_BLOCK) {
188 - sum = red_smem[lane];
189 - }
190 -
191 -// Parallel reduction inside the warp.
192 -#pragma unroll
193 - for (int mask = WARPS_PER_BLOCK / 2; mask >= 1; mask /= 2) {
194 - sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
195 - }
196 -
197 - // Broadcast to other threads.
198 - return __shfl_sync(uint32_t(-1), sum, 0);
199 -}
200 -
162 } // namespace cacheflow
163
164 #undef MMHA_USE_FP32_ACUM_FOR_FMA
csrc/layernorm.cpp new
+14
@@ -0,0 +1,14 @@
1 +#include <torch/extension.h>
2 +
3 +void rms_norm(
4 + torch::Tensor& out,
5 + torch::Tensor& input,
6 + torch::Tensor& weight,
7 + float epsilon);
8 +
9 +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
10 + m.def(
11 + "rms_norm",
12 + &rms_norm,
13 + "Apply Root Mean Square (RMS) Normalization to the input tensor.");
14 +}
csrc/layernorm_kernels.cu new
+61
@@ -0,0 +1,61 @@
1 +#include <torch/extension.h>
2 +#include <ATen/cuda/CUDAContext.h>
3 +
4 +#include "reduction_utils.h"
5 +
6 +namespace cacheflow {
7 +
8 +// TODO(woosuk): Further optimize this kernel.
9 +template<typename scalar_t>
10 +__global__ void rms_norm_kernel(
11 + scalar_t* __restrict__ out, // [num_tokens, hidden_size]
12 + const scalar_t* __restrict__ input, // [num_tokens, hidden_size]
13 + const scalar_t* __restrict__ weight, // [hidden_size]
14 + const float epsilon,
15 + const int num_tokens,
16 + const int hidden_size) {
17 + __shared__ float s_variance;
18 + float variance = 0.0f;
19 +
20 + for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
21 + const float x = (float) input[blockIdx.x * hidden_size + idx];
22 + variance += x * x;
23 + }
24 + variance = blockReduceSum<float>(variance);
25 + if (threadIdx.x == 0) {
26 + s_variance = rsqrtf(variance / hidden_size + epsilon);
27 + }
28 + __syncthreads();
29 +
30 + for (int idx = threadIdx.x; idx < hidden_size; idx += blockDim.x) {
31 + float x = (float) input[blockIdx.x * hidden_size + idx];
32 + out[blockIdx.x * hidden_size + idx] = ((scalar_t) (x * s_variance)) * weight[idx];
33 + }
34 +}
35 +
36 +} // namespace cacheflow
37 +
38 +void rms_norm(
39 + torch::Tensor& out, // [num_tokens, hidden_size]
40 + torch::Tensor& input, // [num_tokens, hidden_size]
41 + torch::Tensor& weight, // [hidden_size]
42 + float epsilon) {
43 + int num_tokens = input.size(0);
44 + int hidden_size = input.size(1);
45 +
46 + dim3 grid(num_tokens);
47 + dim3 block(std::min(hidden_size, 1024));
48 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
49 + AT_DISPATCH_FLOATING_TYPES_AND_HALF(
50 + input.scalar_type(),
51 + "rms_norm_kernel",
52 + [&] {
53 + cacheflow::rms_norm_kernel<scalar_t><<<grid, block, 0, stream>>>(
54 + out.data_ptr<scalar_t>(),
55 + input.data_ptr<scalar_t>(),
56 + weight.data_ptr<scalar_t>(),
57 + epsilon,
58 + num_tokens,
59 + hidden_size);
60 + });
61 +}
csrc/reduction_utils.h new
+76
@@ -0,0 +1,76 @@
1 +#pragma once
2 +
3 +namespace cacheflow {
4 +
5 +template<int WARPS_PER_BLOCK, int WARP_SIZE = 32>
6 +inline __device__ float block_sum(float* red_smem, float sum)
7 +{
8 +
9 + // Decompose the thread index into warp / lane.
10 + int warp = threadIdx.x / WARP_SIZE;
11 + int lane = threadIdx.x % WARP_SIZE;
12 +
13 + // Compute the sum per warp.
14 +#pragma unroll
15 + for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
16 + sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
17 + }
18 +
19 + // Warp leaders store the data to shared memory.
20 + if (lane == 0) {
21 + red_smem[warp] = sum;
22 + }
23 +
24 + // Make sure the data is in shared memory.
25 + __syncthreads();
26 +
27 + // The warps compute the final sums.
28 + if (lane < WARPS_PER_BLOCK) {
29 + sum = red_smem[lane];
30 + }
31 +
32 +// Parallel reduction inside the warp.
33 +#pragma unroll
34 + for (int mask = WARPS_PER_BLOCK / 2; mask >= 1; mask /= 2) {
35 + sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
36 + }
37 +
38 + // Broadcast to other threads.
39 + return __shfl_sync(uint32_t(-1), sum, 0);
40 +}
41 +
42 +#define FINAL_MASK 0xffffffff
43 +
44 +template<typename T>
45 +__inline__ __device__ T warpReduceSum(T val)
46 +{
47 +#pragma unroll
48 + for (int mask = 16; mask > 0; mask >>= 1)
49 + val += __shfl_xor_sync(FINAL_MASK, val, mask, 32);
50 + return val;
51 +}
52 +
53 +/* Calculate the sum of all elements in a block */
54 +template<typename T>
55 +__inline__ __device__ T blockReduceSum(T val)
56 +{
57 + static __shared__ T shared[32];
58 + int lane = threadIdx.x & 0x1f;
59 + int wid = threadIdx.x >> 5;
60 +
61 + val = warpReduceSum<T>(val);
62 +
63 + if (lane == 0)
64 + shared[wid] = val;
65 +
66 + __syncthreads();
67 +
68 + // Modify from blockDim.x << 5 to blockDim.x / 32. to prevent
69 + // blockDim.x is not divided by 32
70 + val = (threadIdx.x < (blockDim.x / 32.f)) ? shared[lane] : (T)(0.0f);
71 + val = warpReduceSum<T>(val);
72 +
73 + return val;
74 +}
75 +
76 +} // namespace cacheflow
setup.py
+8
@@ -31,6 +31,14 @@ positional_encoding_extension = cpp_extension.CUDAExtension(
31 )
32 ext_modules.append(positional_encoding_extension)
33
34 +# Layer normalization kernels.
35 +layernorm_extension = cpp_extension.CUDAExtension(
36 + name='cacheflow.layernorm_ops',
37 + sources=['csrc/layernorm.cpp', 'csrc/layernorm_kernels.cu'],
38 + extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
39 +)
40 +ext_modules.append(layernorm_extension)
41 +
42 setuptools.setup(
43 name='cacheflow',
44 ext_modules=ext_modules,
tests/kernels/layernorm.py new
+53
@@ -0,0 +1,53 @@
1 +import torch
2 +import torch.nn as nn
3 +
4 +from cacheflow import layernorm_ops
5 +
6 +
7 +class RefRMSNorm(nn.Module):
8 +
9 + def __init__(self, hidden_size, eps=1e-6):
10 + super().__init__()
11 + weight = torch.randn(hidden_size) / (hidden_size ** 0.5)
12 + self.weight = nn.Parameter(weight)
13 + self.variance_epsilon = eps
14 +
15 + def forward(self, hidden_states):
16 + variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True)
17 + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
18 + if self.weight.dtype in [torch.half, torch.float16, torch.bfloat16]:
19 + hidden_states = hidden_states.to(self.weight.dtype)
20 + return self.weight * hidden_states
21 +
22 +
23 +@torch.inference_mode()
24 +def test_rms_norm(
25 + num_tokens: int,
26 + hidden_size: int,
27 + dtype: torch.dtype,
28 +) -> None:
29 + x = torch.randn(num_tokens, hidden_size, dtype=dtype, device='cuda')
30 + ref = RefRMSNorm(hidden_size).to(dtype).cuda()
31 +
32 + out = torch.empty_like(x)
33 + layernorm_ops.rms_norm(
34 + out,
35 + x,
36 + ref.weight.data,
37 + ref.variance_epsilon,
38 + )
39 + ref_out = ref(x)
40 + assert torch.allclose(out, ref_out, atol=1e-3, rtol=1e-5)
41 +
42 +
43 +if __name__ == '__main__':
44 + for dtype in [torch.half, torch.float]:
45 + for num_tokens in [7, 128, 2048]:
46 + for hidden_size in [13, 64, 1024, 5120]:
47 + print(f'Testing RMS kernel with dtype={dtype}, num_tokens='
48 + f'{num_tokens}, hidden_size={hidden_size}')
49 + test_rms_norm(
50 + num_tokens=num_tokens,
51 + hidden_size=hidden_size,
52 + dtype=dtype,
53 + )