@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 602358f8a8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add kernel for GeGLU with approximate GELU (#3337)

Woosuk Kwon committed Mar 12, 2024 at 22:06 UTC 602358f8a86ef9fc0ba882e083e19b44e00b9302
5 files changed +49 -7
csrc/activation_kernels.cu
+21 -1
@@ -33,12 +33,25 @@ template<typename T>
33 __device__ __forceinline__ T gelu_kernel(const T& x) {
34 // Equivalent to PyTorch GELU with 'none' approximation.
35 // Refer to:
36 - // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L38
36 + // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L36-L38
37 const float f = (float) x;
38 constexpr float ALPHA = M_SQRT1_2;
39 return (T) (f * 0.5f * (1.0f + ::erf(f * ALPHA)));
40 }
41
42 +template<typename T>
43 +__device__ __forceinline__ T gelu_tanh_kernel(const T& x) {
44 + // Equivalent to PyTorch GELU with 'tanh' approximation.
45 + // Refer to:
46 + // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L25-L30
47 + const float f = (float) x;
48 + constexpr float BETA = M_SQRT2 * M_2_SQRTPI * 0.5f;
49 + constexpr float KAPPA = 0.044715;
50 + float x_cube = f * f * f;
51 + float inner = BETA * (f + KAPPA * x_cube);
52 + return (T) (0.5f * f * (1.0f + ::tanhf(inner)));
53 +}
54 +
55 } // namespace vllm
56
57 // Launch activation and gating kernel.
@@ -73,6 +86,13 @@ void gelu_and_mul(
86 LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel);
87 }
88
89 +void gelu_tanh_and_mul(
90 + torch::Tensor& out, // [..., d]
91 + torch::Tensor& input) // [..., 2 * d]
92 +{
93 + LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_tanh_kernel);
94 +}
95 +
96 namespace vllm {
97
98 // Element-wise activation kernel template.
csrc/ops.h
+4
@@ -61,6 +61,10 @@ void gelu_and_mul(
61 torch::Tensor& out,
62 torch::Tensor& input);
63
64 +void gelu_tanh_and_mul(
65 + torch::Tensor& out,
66 + torch::Tensor& input);
67 +
68 void gelu_new(
69 torch::Tensor& out,
70 torch::Tensor& input);
csrc/pybind.cpp
+5 -1
@@ -25,7 +25,11 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
25 ops.def(
26 "gelu_and_mul",
27 &gelu_and_mul,
28 - "Activation function used in GeGLU.");
28 + "Activation function used in GeGLU with `none` approximation.");
29 + ops.def(
30 + "gelu_tanh_and_mul",
31 + &gelu_tanh_and_mul,
32 + "Activation function used in GeGLU with `tanh` approximation.");
33 ops.def(
34 "gelu_new",
35 &gelu_new,
tests/kernels/test_activation.py
+8 -3
@@ -16,7 +16,7 @@ CUDA_DEVICES = [
16 ]
17
18
19 -@pytest.mark.parametrize("activation", [SiluAndMul, GeluAndMul])
19 +@pytest.mark.parametrize("activation", ["silu", "gelu", "gelu_tanh"])
20 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
21 @pytest.mark.parametrize("d", D)
22 @pytest.mark.parametrize("dtype", DTYPES)
@@ -24,7 +24,7 @@ CUDA_DEVICES = [
24 @pytest.mark.parametrize("device", CUDA_DEVICES)
25 @torch.inference_mode()
26 def test_act_and_mul(
27 - activation: Type[torch.nn.Module],
27 + activation: str,
28 num_tokens: int,
29 d: int,
30 dtype: torch.dtype,
@@ -36,7 +36,12 @@ def test_act_and_mul(
36 torch.cuda.manual_seed(seed)
37 torch.set_default_device(device)
38 x = torch.randn(num_tokens, 2 * d, dtype=dtype)
39 - layer = activation()
39 + if activation == "silu":
40 + layer = SiluAndMul()
41 + elif activation == "gelu":
42 + layer = GeluAndMul(approximate="none")
43 + elif activation == "gelu_tanh":
44 + layer = GeluAndMul(approximate="tanh")
45 out = layer(x)
46 ref_out = layer._forward(x)
47 # The SiLU and GELU implementations are equivalent to the native PyTorch
vllm/model_executor/layers/activation.py
+11 -2
@@ -47,16 +47,25 @@ class GeluAndMul(nn.Module):
47 return: (batch_size, seq_len, d) or (num_tokens, d)
48 """
49
50 + def __init__(self, approximate: str = "none"):
51 + super().__init__()
52 + self.approximate = approximate
53 + if approximate not in ("none", "tanh"):
54 + raise ValueError(f"Unknown approximate mode: {approximate}")
55 +
56 def _forward(self, x: torch.Tensor) -> torch.Tensor:
57 """PyTorch-native implementation equivalent to forward()."""
58 d = x.shape[-1] // 2
53 - return F.gelu(x[..., :d]) * x[..., d:]
59 + return F.gelu(x[..., :d], approximate=self.approximate) * x[..., d:]
60
61 def forward(self, x: torch.Tensor) -> torch.Tensor:
62 d = x.shape[-1] // 2
63 output_shape = (x.shape[:-1] + (d, ))
64 out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
59 - ops.gelu_and_mul(out, x)
65 + if self.approximate == "none":
66 + ops.gelu_and_mul(out, x)
67 + elif self.approximate == "tanh":
68 + ops.gelu_tanh_and_mul(out, x)
69 return out
70
71