@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fd5dcc5c81
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Optimize GeGLU layer in Gemma (#2975)

Woosuk Kwon committed Feb 21, 2024 at 20:17 UTC fd5dcc5c816b7392821d3d4c02b13a7cf820d962
6 files changed +108 -77
csrc/activation_kernels.cu
+48 -25
@@ -2,19 +2,16 @@
2 #include <torch/extension.h>
3 #include <c10/cuda/CUDAGuard.h>
4
5 +#include <cmath>
6 +
7 #include "cuda_compat.h"
8 #include "dispatch_utils.h"
9
10 namespace vllm {
11
10 -template<typename T>
11 -__device__ __forceinline__ T silu(const T& x) {
12 - // x * sigmoid(x)
13 - return (T) (((float) x) / (1.0f + expf((float) -x)));
14 -}
15 -
16 -template<typename scalar_t>
17 -__global__ void silu_and_mul_kernel(
12 +// Activation and gating kernel template.
13 +template<typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
14 +__global__ void act_and_mul_kernel(
15 scalar_t* __restrict__ out, // [..., d]
16 const scalar_t* __restrict__ input, // [..., 2, d]
17 const int d) {
@@ -22,32 +19,58 @@ __global__ void silu_and_mul_kernel(
19 for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
20 const scalar_t x = VLLM_LDG(&input[token_idx * 2 * d + idx]);
21 const scalar_t y = VLLM_LDG(&input[token_idx * 2 * d + d + idx]);
25 - out[token_idx * d + idx] = silu(x) * y;
22 + out[token_idx * d + idx] = ACT_FN(x) * y;
23 }
24 }
25
26 +template<typename T>
27 +__device__ __forceinline__ T silu_kernel(const T& x) {
28 + // x * sigmoid(x)
29 + return (T) (((float) x) / (1.0f + expf((float) -x)));
30 +}
31 +
32 +template<typename T>
33 +__device__ __forceinline__ T gelu_kernel(const T& x) {
34 + // Equivalent to PyTorch GELU with 'none' approximation.
35 + // Refer to:
36 + // https://github.com/pytorch/pytorch/blob/8ac9b20d4b090c213799e81acf48a55ea8d437d6/aten/src/ATen/native/cuda/ActivationGeluKernel.cu#L38
37 + const float f = (float) x;
38 + constexpr float ALPHA = M_SQRT1_2;
39 + return (T) (f * 0.5f * (1.0f + ::erf(f * ALPHA)));
40 +}
41 +
42 } // namespace vllm
43
44 +// Launch activation and gating kernel.
45 +#define LAUNCH_ACTIVATION_GATE_KERNEL(KERNEL) \
46 + int d = input.size(-1) / 2; \
47 + int64_t num_tokens = input.numel() / input.size(-1); \
48 + dim3 grid(num_tokens); \
49 + dim3 block(std::min(d, 1024)); \
50 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
51 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
52 + VLLM_DISPATCH_FLOATING_TYPES( \
53 + input.scalar_type(), \
54 + "act_and_mul_kernel", \
55 + [&] { \
56 + vllm::act_and_mul_kernel<scalar_t, KERNEL<scalar_t>><<<grid, block, 0, stream>>>( \
57 + out.data_ptr<scalar_t>(), \
58 + input.data_ptr<scalar_t>(), \
59 + d); \
60 + });
61 +
62 void silu_and_mul(
63 torch::Tensor& out, // [..., d]
64 torch::Tensor& input) // [..., 2 * d]
65 {
35 - int64_t num_tokens = input.numel() / input.size(-1);
36 - int d = input.size(-1) / 2;
37 -
38 - dim3 grid(num_tokens);
39 - dim3 block(std::min(d, 1024));
40 - const at::cuda::OptionalCUDAGuard device_guard(device_of(input));
41 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
42 - VLLM_DISPATCH_FLOATING_TYPES(
43 - input.scalar_type(),
44 - "silu_and_mul_kernel",
45 - [&] {
46 - vllm::silu_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
47 - out.data_ptr<scalar_t>(),
48 - input.data_ptr<scalar_t>(),
49 - d);
50 - });
66 + LAUNCH_ACTIVATION_GATE_KERNEL(vllm::silu_kernel);
67 +}
68 +
69 +void gelu_and_mul(
70 + torch::Tensor& out, // [..., d]
71 + torch::Tensor& input) // [..., 2 * d]
72 +{
73 + LAUNCH_ACTIVATION_GATE_KERNEL(vllm::gelu_kernel);
74 }
75
76 namespace vllm {
csrc/ops.h
+4
@@ -57,6 +57,10 @@ void silu_and_mul(
57 torch::Tensor& out,
58 torch::Tensor& input);
59
60 +void gelu_and_mul(
61 + torch::Tensor& out,
62 + torch::Tensor& input);
63 +
64 void gelu_new(
65 torch::Tensor& out,
66 torch::Tensor& input);
csrc/pybind.cpp
+4
@@ -22,6 +22,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
22 "silu_and_mul",
23 &silu_and_mul,
24 "Activation function used in SwiGLU.");
25 + ops.def(
26 + "gelu_and_mul",
27 + &gelu_and_mul,
28 + "Activation function used in GeGLU.");
29 ops.def(
30 "gelu_new",
31 &gelu_new,
tests/kernels/test_activation.py
+15 -35
@@ -1,7 +1,10 @@
1 +from typing import Type
2 +
3 import pytest
4 import torch
5
4 -from vllm.model_executor.layers.activation import FastGELU, NewGELU, SiluAndMul
6 +from vllm.model_executor.layers.activation import (FastGELU, GeluAndMul,
7 + NewGELU, SiluAndMul)
8 from allclose_default import get_default_atol, get_default_rtol
9
10 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -13,13 +16,15 @@ CUDA_DEVICES = [
16 ]
17
18
19 +@pytest.mark.parametrize("activation", [SiluAndMul, GeluAndMul])
20 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
21 @pytest.mark.parametrize("d", D)
22 @pytest.mark.parametrize("dtype", DTYPES)
23 @pytest.mark.parametrize("seed", SEEDS)
24 @pytest.mark.parametrize("device", CUDA_DEVICES)
25 @torch.inference_mode()
22 -def test_silu_and_mul(
26 +def test_act_and_mul(
27 + activation: Type[torch.nn.Module],
28 num_tokens: int,
29 d: int,
30 dtype: torch.dtype,
@@ -31,48 +36,23 @@ def test_silu_and_mul(
36 torch.cuda.manual_seed(seed)
37 torch.set_default_device(device)
38 x = torch.randn(num_tokens, 2 * d, dtype=dtype)
34 - layer = SiluAndMul()
39 + layer = activation()
40 out = layer(x)
41 ref_out = layer._forward(x)
37 - assert torch.allclose(out,
38 - ref_out,
39 - atol=get_default_atol(out),
40 - rtol=get_default_rtol(out))
42 + # The SiLU and GELU implementations are equivalent to the native PyTorch
43 + # implementations, so we can do exact comparison.
44 + assert torch.allclose(out, ref_out, atol=0.0, rtol=0.0)
45
46
47 +@pytest.mark.parametrize("activation", [FastGELU, NewGELU])
48 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
49 @pytest.mark.parametrize("d", D)
50 @pytest.mark.parametrize("dtype", DTYPES)
51 @pytest.mark.parametrize("seed", SEEDS)
52 @pytest.mark.parametrize("device", CUDA_DEVICES)
53 @torch.inference_mode()
49 -def test_gelu_new(
50 - num_tokens: int,
51 - d: int,
52 - dtype: torch.dtype,
53 - seed: int,
54 - device: str,
55 -) -> None:
56 - torch.random.manual_seed(seed)
57 - if torch.cuda.is_available():
58 - torch.cuda.manual_seed(seed)
59 - torch.set_default_device(device)
60 - x = torch.randn(num_tokens, d, dtype=dtype)
61 - layer = NewGELU()
62 - out = layer(x)
63 - ref_out = layer._forward(x)
64 - assert torch.allclose(out,
65 - ref_out,
66 - atol=get_default_atol(out),
67 - rtol=get_default_rtol(out))
68 -
69 -
70 -@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
71 -@pytest.mark.parametrize("d", D)
72 -@pytest.mark.parametrize("dtype", DTYPES)
73 -@pytest.mark.parametrize("seed", SEEDS)
74 -@pytest.mark.parametrize("device", CUDA_DEVICES)
75 -def test_gelu_fast(
54 +def test_activation(
55 + activation: Type[torch.nn.Module],
56 num_tokens: int,
57 d: int,
58 dtype: torch.dtype,
@@ -84,7 +64,7 @@ def test_gelu_fast(
64 torch.cuda.manual_seed(seed)
65 torch.set_default_device(device)
66 x = torch.randn(num_tokens, d, dtype=dtype)
87 - layer = FastGELU()
67 + layer = activation()
68 out = layer(x)
69 ref_out = layer._forward(x)
70 assert torch.allclose(out,
vllm/model_executor/layers/activation.py
+23
@@ -37,6 +37,29 @@ class SiluAndMul(nn.Module):
37 return out
38
39
40 +class GeluAndMul(nn.Module):
41 + """An activation function for GeGLU.
42 +
43 + The function computes x -> GELU(x[:d]) * x[d:] where d = x.shape[-1] // 2.
44 +
45 + Shapes:
46 + x: (batch_size, seq_len, 2 * d) or (num_tokens, 2 * d)
47 + return: (batch_size, seq_len, d) or (num_tokens, d)
48 + """
49 +
50 + def _forward(self, x: torch.Tensor) -> torch.Tensor:
51 + """PyTorch-native implementation equivalent to forward()."""
52 + d = x.shape[-1] // 2
53 + return F.gelu(x[..., :d]) * x[..., d:]
54 +
55 + def forward(self, x: torch.Tensor) -> torch.Tensor:
56 + d = x.shape[-1] // 2
57 + output_shape = (x.shape[:-1] + (d, ))
58 + out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
59 + ops.gelu_and_mul(out, x)
60 + return out
61 +
62 +
63 class NewGELU(nn.Module):
64
65 def _forward(self, x: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/gemma.py
+14 -17
@@ -21,10 +21,11 @@ from torch import nn
21 from transformers import GemmaConfig
22
23 from vllm.model_executor.input_metadata import InputMetadata
24 +from vllm.model_executor.layers.activation import GeluAndMul
25 from vllm.model_executor.layers.attention import PagedAttention
26 from vllm.model_executor.layers.layernorm import RMSNorm
26 -from vllm.model_executor.layers.linear import (ColumnParallelLinear,
27 - LinearMethodBase,
27 +from vllm.model_executor.layers.linear import (LinearMethodBase,
28 + MergedColumnParallelLinear,
29 QKVParallelLinear,
30 RowParallelLinear)
31 from vllm.model_executor.layers.rotary_embedding import get_rope
@@ -50,27 +51,21 @@ class GemmaMLP(nn.Module):
51 linear_method: Optional[LinearMethodBase] = None,
52 ) -> None:
53 super().__init__()
53 - self.gate_proj = ColumnParallelLinear(hidden_size,
54 - intermediate_size,
55 - bias=False,
56 - linear_method=linear_method)
57 - self.up_proj = ColumnParallelLinear(hidden_size,
58 - intermediate_size,
59 - bias=False,
60 - linear_method=linear_method)
54 + self.gate_up_proj = MergedColumnParallelLinear(
55 + hidden_size, [intermediate_size] * 2,
56 + bias=False,
57 + linear_method=linear_method)
58 self.down_proj = RowParallelLinear(intermediate_size,
59 hidden_size,
60 bias=False,
61 linear_method=linear_method)
65 - self.act_fn = nn.GELU()
62 + self.act_fn = GeluAndMul()
63
64 def forward(self, x):
68 - gate, _ = self.gate_proj(x)
69 - gate = self.act_fn(gate)
70 - up, _ = self.up_proj(x)
71 - fuse = gate * up
72 - outputs, _ = self.down_proj(fuse)
73 - return outputs
65 + gate_up, _ = self.gate_up_proj(x)
66 + x = self.act_fn(gate_up)
67 + x, _ = self.down_proj(x)
68 + return x
69
70
71 class GemmaAttention(nn.Module):
@@ -294,6 +289,8 @@ class GemmaForCausalLM(nn.Module):
289 ("qkv_proj", "q_proj", "q"),
290 ("qkv_proj", "k_proj", "k"),
291 ("qkv_proj", "v_proj", "v"),
292 + ("gate_up_proj", "gate_proj", 0),
293 + ("gate_up_proj", "up_proj", 1),
294 ]
295 params_dict = dict(self.named_parameters())
296 loaded_params = set()