@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d64bf1646c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Implement approximate GELU kernels (#828)

Woosuk Kwon committed Aug 23, 2023 at 07:43 UTC d64bf1646c19123bf236d5ba1c02b75ee3d68306
4 files changed +164 -18
csrc/activation.cpp
+16
@@ -4,9 +4,25 @@ void silu_and_mul(
4 torch::Tensor& out,
5 torch::Tensor& input);
6
7 +void gelu_new(
8 + torch::Tensor& out,
9 + torch::Tensor& input);
10 +
11 +void gelu_fast(
12 + torch::Tensor& out,
13 + torch::Tensor& input);
14 +
15 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
16 m.def(
17 "silu_and_mul",
18 &silu_and_mul,
19 "Activation function used in SwiGLU.");
20 + m.def(
21 + "gelu_new",
22 + &gelu_new,
23 + "GELU implementation used in GPT-2.");
24 + m.def(
25 + "gelu_fast",
26 + &gelu_fast,
27 + "Approximate GELU implementation.");
28 }
csrc/activation_kernels.cu
+68
@@ -46,3 +46,71 @@ void silu_and_mul(
46 d);
47 });
48 }
49 +
50 +namespace vllm {
51 +
52 +// Element-wise activation kernel template.
53 +template<typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
54 +__global__ void activation_kernel(
55 + scalar_t* __restrict__ out, // [num_tokens, d]
56 + const scalar_t* __restrict__ input, // [num_tokens, d]
57 + const int d) {
58 + const int token_idx = blockIdx.x;
59 + for (int idx = threadIdx.x; idx < d; idx += blockDim.x) {
60 + const scalar_t x = __ldg(&input[token_idx * d + idx]);
61 + out[token_idx * d + idx] = ACT_FN(x);
62 + }
63 +}
64 +
65 +} // namespace vllm
66 +
67 +// Launch element-wise activation kernel.
68 +#define LAUNCH_ACTIVATION_KERNEL(KERNEL) \
69 + int num_tokens = input.size(0); \
70 + int d = input.size(1); \
71 + dim3 grid(num_tokens); \
72 + dim3 block(std::min(d, 1024)); \
73 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
74 + AT_DISPATCH_FLOATING_TYPES_AND2( \
75 + at::ScalarType::Half, \
76 + at::ScalarType::BFloat16, \
77 + input.scalar_type(), \
78 + "activation_kernel", \
79 + [&] { \
80 + vllm::activation_kernel<scalar_t, KERNEL<scalar_t>><<<grid, block, 0, stream>>>( \
81 + out.data_ptr<scalar_t>(), \
82 + input.data_ptr<scalar_t>(), \
83 + d); \
84 + });
85 +
86 +namespace vllm {
87 +
88 +template<typename T>
89 +__device__ __forceinline__ T gelu_new_kernel(const T& x) {
90 + const float x3 = (float) (x * x * x);
91 + const T t = (T) tanhf((T) (0.79788456f * (float) (x + (T) (0.044715f * x3))));
92 + return ((T) 0.5) * x * (((T) 1.0) + t);
93 +}
94 +
95 +template<typename T>
96 +__device__ __forceinline__ T gelu_fast_kernel(const T& x) {
97 + const float f = (float) x;
98 + const T t = (T) tanhf(((T) (f * 0.79788456f)) * (((T) 1.0) + (T) (0.044715f * f) * x));
99 + return ((T) 0.5) * x * (((T) 1.0) + t);
100 +}
101 +
102 +} // namespace vllm
103 +
104 +void gelu_new(
105 + torch::Tensor& out, // [num_tokens, d]
106 + torch::Tensor& input) // [num_tokens, d]
107 +{
108 + LAUNCH_ACTIVATION_KERNEL(vllm::gelu_new_kernel);
109 +}
110 +
111 +void gelu_fast(
112 + torch::Tensor& out, // [num_tokens, d]
113 + torch::Tensor& input) // [num_tokens, d]
114 +{
115 + LAUNCH_ACTIVATION_KERNEL(vllm::gelu_fast_kernel);
116 +}
tests/kernels/test_activation.py
+43 -1
@@ -1,6 +1,6 @@
1 import torch
2 import torch.nn.functional as F
3 -
3 +from transformers.activations import get_activation
4 from vllm import activation_ops
5
6
@@ -28,3 +28,45 @@ def test_silu_and_mul() -> None:
28 for d in [512, 4096, 5120, 13824]:
29 print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
30 run_silu_and_mul(num_tokens, d, dtype)
31 +
32 +
33 +@torch.inference_mode()
34 +def run_gelu_new(
35 + num_tokens: int,
36 + d: int,
37 + dtype: torch.dtype,
38 +) -> None:
39 + x = torch.randn(num_tokens, d, dtype=dtype, device='cuda')
40 + out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
41 + activation_ops.gelu_new(out, x)
42 + ref_out = get_activation("gelu_new")(x)
43 + assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
44 +
45 +
46 +def test_gelu_new() -> None:
47 + for dtype in [torch.half, torch.bfloat16, torch.float]:
48 + for num_tokens in [7, 83, 2048]:
49 + for d in [512, 4096, 5120, 13824]:
50 + print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
51 + run_gelu_new(num_tokens, d, dtype)
52 +
53 +
54 +@torch.inference_mode()
55 +def run_gelu_fast(
56 + num_tokens: int,
57 + d: int,
58 + dtype: torch.dtype,
59 +) -> None:
60 + x = torch.randn(num_tokens, d, dtype=dtype, device='cuda')
61 + out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
62 + activation_ops.gelu_fast(out, x)
63 + ref_out = get_activation("gelu_fast")(x)
64 + assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
65 +
66 +
67 +def test_gelu_fast() -> None:
68 + for dtype in [torch.half, torch.bfloat16, torch.float]:
69 + for num_tokens in [7, 83, 2048]:
70 + for d in [512, 4096, 5120, 13824]:
71 + print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
72 + run_gelu_fast(num_tokens, d, dtype)
vllm/model_executor/layers/activation.py
+37 -17
@@ -4,23 +4,6 @@ import torch.nn as nn
4
5 from vllm import activation_ops
6
7 -_ACTIVATION_REGISTRY = {
8 - "gelu": nn.GELU(),
9 - # NOTE: The following GELU functions may introduce small rounding errors.
10 - "gelu_new": nn.GELU(approximate="tanh"),
11 - "gelu_fast": nn.GELU(approximate="tanh"),
12 - "gelu_pytorch_tanh": nn.GELU(approximate="tanh"),
13 - "relu": nn.ReLU(),
14 -}
15 -
16 -
17 -def get_act_fn(act_fn: str) -> nn.Module:
18 - """Get an activation function by name."""
19 - act_fn = act_fn.lower()
20 - if act_fn in _ACTIVATION_REGISTRY:
21 - return _ACTIVATION_REGISTRY[act_fn]
22 - raise ValueError(f"Activation function {act_fn!r} is not supported.")
23 -
7
8 class SiluAndMul(nn.Module):
9 """An activation function for SwiGLU.
@@ -38,3 +21,40 @@ class SiluAndMul(nn.Module):
21 out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
22 activation_ops.silu_and_mul(out, x)
23 return out
24 +
25 +
26 +class NewGELU(nn.Module):
27 +
28 + def forward(self, x: torch.Tensor) -> torch.Tensor:
29 + num_tokens = x.shape[0]
30 + d = x.shape[1]
31 + out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
32 + activation_ops.gelu_new(out, x)
33 + return out
34 +
35 +
36 +class FastGELU(nn.Module):
37 +
38 + def forward(self, x: torch.Tensor) -> torch.Tensor:
39 + num_tokens = x.shape[0]
40 + d = x.shape[1]
41 + out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
42 + activation_ops.gelu_fast(out, x)
43 + return out
44 +
45 +
46 +_ACTIVATION_REGISTRY = {
47 + "gelu": nn.GELU(),
48 + "gelu_fast": FastGELU(),
49 + "gelu_new": NewGELU(),
50 + "gelu_pytorch_tanh": nn.GELU(approximate="tanh"),
51 + "relu": nn.ReLU(),
52 +}
53 +
54 +
55 +def get_act_fn(act_fn: str) -> nn.Module:
56 + """Get an activation function by name."""
57 + act_fn = act_fn.lower()
58 + if act_fn in _ACTIVATION_REGISTRY:
59 + return _ACTIVATION_REGISTRY[act_fn]
60 + raise ValueError(f"Activation function {act_fn!r} is not supported.")