A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add kernel for GeGLU with approximate GELU (#3337)
Woosuk Kwon committed
Mar 12, 2024 at 22:06 UTC
602358f8a86ef9fc0ba882e083e19b44e00b9302
5 files changed
+49
-7
csrc/activation_kernels.cu
+21
-1
@@ -33,12 +33,25 @@ template<typename T>
33
__device__ __forceinline__ T gelu_kernel(const T& x) {
34
// Equivalent to PyTorch GELU with 'none' approximation.
35
// Refer to:
36
- // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L38
36
+ // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L36-L38
37
const float f = (float) x;
38
constexpr float ALPHA = M_SQRT1_2;
39
return (T) (f * 0.5f * (1.0f + ::erf(f * ALPHA)));
40
}
41
42
+template<typename T>
43
+__device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
44
+ // Equivalent to PyTorch GELU with 'tanh' approximation.
45
+ // Refer to:
46
+ // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L25-L30
47
+ const float f = (float) x;
48
+ constexpr float BETA = M_SQRT2 * M_2_SQRTPI * 0.5f;
49
+ constexpr float KAPPA = 0.044715;
50
+ float x_cube = f * f * f;
51
+ float inner = BETA * (f + KAPPA * x_cube);
52
+ return (T) (0.5f * f * (1.0f + ::tanhf(inner)));
53
+}
54
+
55
} // namespace vllm
56
57
// Launch activation and gating kernel.
@@ -73,6 +86,13 @@ void gelu_and_mul(
86
LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel);
87
}
88
89
+void gelu_tanh_and_mul(
90
+ torch::Tensor& out, // [..., d]
91
+ torch::Tensor& input) // [..., 2 * d]
92
+{
93
+ LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_tanh_kernel);
94
+}
95
+
96
namespace vllm {
97
98
// Element-wise activation kernel template.
csrc/ops.h
+4
@@ -61,6 +61,10 @@ void gelu_and_mul(
61
torch::Tensor& out,
62
torch::Tensor& input);
63
64
+void gelu_tanh_and_mul(
65
+ torch::Tensor& out,
66
+ torch::Tensor& input);
67
+
68
void gelu_new(
69
torch::Tensor& out,
70
torch::Tensor& input);
csrc/pybind.cpp
+5
-1
@@ -25,7 +25,11 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
25
ops.def(
26
"gelu_and_mul",
27
&gelu_and_mul,
28
- "Activation function used in GeGLU.");
28
+ "Activation function used in GeGLU with `none` approximation.");
29
+ ops.def(
30
+ "gelu_tanh_and_mul",
31
+ &gelu_tanh_and_mul,
32
+ "Activation function used in GeGLU with `tanh` approximation.");
33
ops.def(
34
"gelu_new",
35
&gelu_new,
tests/kernels/test_activation.py
+8
-3
@@ -16,7 +16,7 @@ CUDA_DEVICES = [
16
]
17
18
19
-@pytest.mark.parametrize("activation", [SiluAndMul, GeluAndMul])
19
+@pytest.mark.parametrize("activation", ["silu", "gelu", "gelu_tanh"])
20
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
21
@pytest.mark.parametrize("d", D)
22
@pytest.mark.parametrize("dtype", DTYPES)
@@ -24,7 +24,7 @@ CUDA_DEVICES = [
24
@pytest.mark.parametrize("device", CUDA_DEVICES)
25
@torch.inference_mode()
26
def test_act_and_mul(
27
- activation: Type[torch.nn.Module],
27
+ activation: str,
28
num_tokens: int,
29
d: int,
30
dtype: torch.dtype,
@@ -36,7 +36,12 @@ def test_act_and_mul(
36
torch.cuda.manual_seed(seed)
37
torch.set_default_device(device)
38
x = torch.randn(num_tokens, 2 * d, dtype=dtype)
39
- layer = activation()
39
+ if activation == "silu":
40
+ layer = SiluAndMul()
41
+ elif activation == "gelu":
42
+ layer = GeluAndMul(approximate="none")
43
+ elif activation == "gelu_tanh":
44
+ layer = GeluAndMul(approximate="tanh")
45
out = layer(x)
46
ref_out = layer._forward(x)
47
# The SiLU and GELU implementations are equivalent to the native PyTorch
vllm/model_executor/layers/activation.py
+11
-2
@@ -47,16 +47,25 @@ class GeluAndMul(nn.Module):
47
return: (batch_size, seq_len, d) or (num_tokens, d)
48
"""
49
50
+ def __init__(self, approximate: str = "none"):
51
+ super().__init__()
52
+ self.approximate = approximate
53
+ if approximate not in ("none", "tanh"):
54
+ raise ValueError(f"Unknown approximate mode: {approximate}")
55
+
56
def _forward(self, x: torch.Tensor) -> torch.Tensor:
57
"""PyTorch-native implementation equivalent to forward()."""
58
d = x.shape[-1] // 2
53
- return F.gelu(x[..., :d]) * x[..., d:]
59
+ return F.gelu(x[..., :d], approximate=self.approximate) * x[..., d:]
60
61
def forward(self, x: torch.Tensor) -> torch.Tensor:
62
d = x.shape[-1] // 2
63
output_shape = (x.shape[:-1] + (d, ))
64
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
59
- ops.gelu_and_mul(out, x)
65
+ if self.approximate == "none":
66
+ ops.gelu_and_mul(out, x)
67
+ elif self.approximate == "tanh":
68
+ ops.gelu_tanh_and_mul(out, x)
69
return out
70
71