A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Optimize GeGLU layer in Gemma (#2975)
Woosuk Kwon committed
Feb 21, 2024 at 20:17 UTC
fd5dcc5c816b7392821d3d4c02b13a7cf820d962
6 files changed
+108
-77
csrc/activation_kernels.cu
+48
-25
@@ -2,19 +2,16 @@
2
#include <torch/extension.h>
3
#include <c10/cuda/CUDAGuard.h>
4
5
+#include <cmath>
6
+
7
#include "cuda_compat.h"
8
#include "dispatch_utils.h"
9
10
namespace vllm {
11
10
-template<typename T>
11
-__device__ __forceinline__ T silu(const T& x) {
12
- // x * sigmoid(x)
13
- return (T) (((float) x) / (1.0f + expf((float) -x)));
14
-}
15
-
16
-template<typename scalar_t>
17
-__global__ void silu_and_mul_kernel(
12
+// Activation and gating kernel template.
13
+template<typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
14
+__global__ void act_and_mul_kernel(
15
scalar_t* __restrict__ out, // [..., d]
16
const scalar_t* __restrict__ input, // [..., 2, d]
17
const int d) {
@@ -22,32 +19,58 @@ __global__ void silu_and_mul_kernel(
19
for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
20
const scalar_t x = VLLM_LDG(&input[token_idx * 2 * d + idx]);
21
const scalar_t y = VLLM_LDG(&input[token_idx * 2 * d + d + idx]);
25
- out[token_idx * d + idx] = silu(x) * y;
22
+ out[token_idx * d + idx] = ACT_FN(x) * y;
23
}
24
}
25
26
+template<typename T>
27
+__device__ __forceinline__ T silu_kernel(const T& x) {
28
+ // x * sigmoid(x)
29
+ return (T) (((float) x) / (1.0f + expf((float) -x)));
30
+}
31
+
32
+template<typename T>
33
+__device__ __forceinline__ T gelu_kernel(const T& x) {
34
+ // Equivalent to PyTorch GELU with 'none' approximation.
35
+ // Refer to:
36
+ // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L38
37
+ const float f = (float) x;
38
+ constexpr float ALPHA = M_SQRT1_2;
39
+ return (T) (f * 0.5f * (1.0f + ::erf(f * ALPHA)));
40
+}
41
+
42
} // namespace vllm
43
44
+// Launch activation and gating kernel.
45
+#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL) \
46
+ int d = input.size(-1) / 2; \
47
+ int64_t num_tokens = input.numel() / input.size(-1); \
48
+ dim3 grid(num_tokens); \
49
+ dim3 block(std::min(d, 1024)); \
50
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
51
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
52
+ VLLM_DISPATCH_FLOATING_TYPES( \
53
+ input.scalar_type(), \
54
+ "act_and_mul_kernel", \
55
+ [&] { \
56
+ vllm::act_and_mul_kernel<scalar_t, KERNEL<scalar_t>><<<grid, block, 0, stream>>>( \
57
+ out.data_ptr<scalar_t>(), \
58
+ input.data_ptr<scalar_t>(), \
59
+ d); \
60
+ });
61
+
62
void silu_and_mul(
63
torch::Tensor& out, // [..., d]
64
torch::Tensor& input) // [..., 2 * d]
65
{
35
- int64_t num_tokens = input.numel() / input.size(-1);
36
- int d = input.size(-1) / 2;
37
-
38
- dim3 grid(num_tokens);
39
- dim3 block(std::min(d, 1024));
40
- const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
41
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
42
- VLLM_DISPATCH_FLOATING_TYPES(
43
- input.scalar_type(),
44
- "silu_and_mul_kernel",
45
- [&] {
46
- vllm::silu_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
47
- out.data_ptr<scalar_t>(),
48
- input.data_ptr<scalar_t>(),
49
- d);
50
- });
66
+ LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel);
67
+}
68
+
69
+void gelu_and_mul(
70
+ torch::Tensor& out, // [..., d]
71
+ torch::Tensor& input) // [..., 2 * d]
72
+{
73
+ LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel);
74
}
75
76
namespace vllm {
csrc/ops.h
+4
@@ -57,6 +57,10 @@ void silu_and_mul(
57
torch::Tensor& out,
58
torch::Tensor& input);
59
60
+void gelu_and_mul(
61
+ torch::Tensor& out,
62
+ torch::Tensor& input);
63
+
64
void gelu_new(
65
torch::Tensor& out,
66
torch::Tensor& input);
csrc/pybind.cpp
+4
@@ -22,6 +22,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
22
"silu_and_mul",
23
&silu_and_mul,
24
"Activation function used in SwiGLU.");
25
+ ops.def(
26
+ "gelu_and_mul",
27
+ &gelu_and_mul,
28
+ "Activation function used in GeGLU.");
29
ops.def(
30
"gelu_new",
31
&gelu_new,
tests/kernels/test_activation.py
+15
-35
@@ -1,7 +1,10 @@
1
+from typing import Type
2
+
3
import pytest
4
import torch
5
4
-from vllm.model_executor.layers.activation import FastGELU, NewGELU, SiluAndMul
6
+from vllm.model_executor.layers.activation import (FastGELU, GeluAndMul,
7
+ NewGELU, SiluAndMul)
8
from allclose_default import get_default_atol, get_default_rtol
9
10
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -13,13 +16,15 @@ CUDA_DEVICES = [
16
]
17
18
19
+@pytest.mark.parametrize("activation", [SiluAndMul, GeluAndMul])
20
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
21
@pytest.mark.parametrize("d", D)
22
@pytest.mark.parametrize("dtype", DTYPES)
23
@pytest.mark.parametrize("seed", SEEDS)
24
@pytest.mark.parametrize("device", CUDA_DEVICES)
25
@torch.inference_mode()
22
-def test_silu_and_mul(
26
+def test_act_and_mul(
27
+ activation: Type[torch.nn.Module],
28
num_tokens: int,
29
d: int,
30
dtype: torch.dtype,
@@ -31,48 +36,23 @@ def test_silu_and_mul(
36
torch.cuda.manual_seed(seed)
37
torch.set_default_device(device)
38
x = torch.randn(num_tokens, 2 * d, dtype=dtype)
34
- layer = SiluAndMul()
39
+ layer = activation()
40
out = layer(x)
41
ref_out = layer._forward(x)
37
- assert torch.allclose(out,
38
- ref_out,
39
- atol=get_default_atol(out),
40
- rtol=get_default_rtol(out))
42
+ # The SiLU and GELU implementations are equivalent to the native PyTorch
43
+ # implementations, so we can do exact comparison.
44
+ assert torch.allclose(out, ref_out, atol=0.0, rtol=0.0)
45
46
47
+@pytest.mark.parametrize("activation", [FastGELU, NewGELU])
48
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
49
@pytest.mark.parametrize("d", D)
50
@pytest.mark.parametrize("dtype", DTYPES)
51
@pytest.mark.parametrize("seed", SEEDS)
52
@pytest.mark.parametrize("device", CUDA_DEVICES)
53
@torch.inference_mode()
49
-def test_gelu_new(
50
- num_tokens: int,
51
- d: int,
52
- dtype: torch.dtype,
53
- seed: int,
54
- device: str,
55
-) -> None:
56
- torch.random.manual_seed(seed)
57
- if torch.cuda.is_available():
58
- torch.cuda.manual_seed(seed)
59
- torch.set_default_device(device)
60
- x = torch.randn(num_tokens, d, dtype=dtype)
61
- layer = NewGELU()
62
- out = layer(x)
63
- ref_out = layer._forward(x)
64
- assert torch.allclose(out,
65
- ref_out,
66
- atol=get_default_atol(out),
67
- rtol=get_default_rtol(out))
68
-
69
-
70
-@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
71
-@pytest.mark.parametrize("d", D)
72
-@pytest.mark.parametrize("dtype", DTYPES)
73
-@pytest.mark.parametrize("seed", SEEDS)
74
-@pytest.mark.parametrize("device", CUDA_DEVICES)
75
-def test_gelu_fast(
54
+def test_activation(
55
+ activation: Type[torch.nn.Module],
56
num_tokens: int,
57
d: int,
58
dtype: torch.dtype,
@@ -84,7 +64,7 @@ def test_gelu_fast(
64
torch.cuda.manual_seed(seed)
65
torch.set_default_device(device)
66
x = torch.randn(num_tokens, d, dtype=dtype)
87
- layer = FastGELU()
67
+ layer = activation()
68
out = layer(x)
69
ref_out = layer._forward(x)
70
assert torch.allclose(out,
vllm/model_executor/layers/activation.py
+23
@@ -37,6 +37,29 @@ class SiluAndMul(nn.Module):
37
return out
38
39
40
+class GeluAndMul(nn.Module):
41
+ """An activation function for GeGLU.
42
+
43
+ The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2.
44
+
45
+ Shapes:
46
+ x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d)
47
+ return: (batch_size, seq_len, d) or (num_tokens, d)
48
+ """
49
+
50
+ def _forward(self, x: torch.Tensor) -> torch.Tensor:
51
+ """PyTorch-native implementation equivalent to forward()."""
52
+ d = x.shape[-1] // 2
53
+ return F.gelu(x[..., :d]) * x[..., d:]
54
+
55
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
56
+ d = x.shape[-1] // 2
57
+ output_shape = (x.shape[:-1] + (d, ))
58
+ out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
59
+ ops.gelu_and_mul(out, x)
60
+ return out
61
+
62
+
63
class NewGELU(nn.Module):
64
65
def _forward(self, x: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/gemma.py
+14
-17
@@ -21,10 +21,11 @@ from torch import nn
21
from transformers import GemmaConfig
22
23
from vllm.model_executor.input_metadata import InputMetadata
24
+from vllm.model_executor.layers.activation import GeluAndMul
25
from vllm.model_executor.layers.attention import PagedAttention
26
from vllm.model_executor.layers.layernorm import RMSNorm
26
-from vllm.model_executor.layers.linear import (ColumnParallelLinear,
27
- LinearMethodBase,
27
+from vllm.model_executor.layers.linear import (LinearMethodBase,
28
+ MergedColumnParallelLinear,
29
QKVParallelLinear,
30
RowParallelLinear)
31
from vllm.model_executor.layers.rotary_embedding import get_rope
@@ -50,27 +51,21 @@ class GemmaMLP(nn.Module):
51
linear_method: Optional[LinearMethodBase] = None,
52
) -> None:
53
super().__init__()
53
- self.gate_proj = ColumnParallelLinear(hidden_size,
54
- intermediate_size,
55
- bias=False,
56
- linear_method=linear_method)
57
- self.up_proj = ColumnParallelLinear(hidden_size,
58
- intermediate_size,
59
- bias=False,
60
- linear_method=linear_method)
54
+ self.gate_up_proj = MergedColumnParallelLinear(
55
+ hidden_size, [intermediate_size] * 2,
56
+ bias=False,
57
+ linear_method=linear_method)
58
self.down_proj = RowParallelLinear(intermediate_size,
59
hidden_size,
60
bias=False,
61
linear_method=linear_method)
65
- self.act_fn = nn.GELU()
62
+ self.act_fn = GeluAndMul()
63
64
def forward(self, x):
68
- gate, _ = self.gate_proj(x)
69
- gate = self.act_fn(gate)
70
- up, _ = self.up_proj(x)
71
- fuse = gate * up
72
- outputs, _ = self.down_proj(fuse)
73
- return outputs
65
+ gate_up, _ = self.gate_up_proj(x)
66
+ x = self.act_fn(gate_up)
67
+ x, _ = self.down_proj(x)
68
+ return x
69
70
71
class GemmaAttention(nn.Module):
@@ -294,6 +289,8 @@ class GemmaForCausalLM(nn.Module):
289
("qkv_proj", "q_proj", "q"),
290
("qkv_proj", "k_proj", "k"),
291
("qkv_proj", "v_proj", "v"),
292
+ ("gate_up_proj", "gate_proj", 0),
293
+ ("gate_up_proj", "up_proj", 1),
294
]
295
params_dict = dict(self.named_parameters())
296
loaded_params = set()