@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 295a061fb3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Kernel] add kernel for FATReLU (#9610)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Oct 24, 2024 at 16:18 UTC 295a061fb34ec6fb251abf1dbece5b1bb7dc9006
6 files changed +78 -8
csrc/activation_kernels.cu
+42
@@ -89,6 +89,48 @@ void gelu_tanh_and_mul(torch::Tensor& out, // [..., d]
89
90 namespace vllm {
91
92 +template <typename T>
93 +__device__ __forceinline__ T fatrelu_kernel(const T& x, const float threshold) {
94 + const float f = (float)x;
95 + return (T)(f > threshold ? f : 0.0f);
96 +}
97 +
98 +template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&, const float)>
99 +__global__ void act_and_mul_kernel_with_param(
100 + scalar_t* __restrict__ out, const scalar_t* __restrict__ input, const int d,
101 + const float param) {
102 + const int64_t token_idx = blockIdx.x;
103 + for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
104 + const scalar_t x = VLLM_LDG(&input[token_idx * 2 * d + idx]);
105 + const scalar_t y = VLLM_LDG(&input[token_idx * 2 * d + d + idx]);
106 + out[token_idx * d + idx] = ACT_FN(x, param) * y;
107 + }
108 +}
109 +
110 +} // namespace vllm
111 +
112 +#define LAUNCH_ACTIVATION_GATE_KERNEL_WITH_PARAM(KERNEL, PARAM) \
113 + int d = input.size(-1) / 2; \
114 + int64_t num_tokens = input.numel() / input.size(-1); \
115 + dim3 grid(num_tokens); \
116 + dim3 block(std::min(d, 1024)); \
117 + const at::cuda::OptionalCUDAGuard device_guard(device_of(input)); \
118 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
119 + VLLM_DISPATCH_FLOATING_TYPES( \
120 + input.scalar_type(), "act_and_mul_kernel_with_param", [&] { \
121 + vllm::act_and_mul_kernel_with_param<scalar_t, KERNEL<scalar_t>> \
122 + <<<grid, block, 0, stream>>>(out.data_ptr<scalar_t>(), \
123 + input.data_ptr<scalar_t>(), d, \
124 + PARAM); \
125 + });
126 +
127 +void fatrelu_and_mul(torch::Tensor& out, // [..., d],
128 + torch::Tensor& input, // [..., 2 * d]
129 + double threshold) {
130 + LAUNCH_ACTIVATION_GATE_KERNEL_WITH_PARAM(vllm::fatrelu_kernel, threshold);
131 +}
132 +namespace vllm {
133 +
134 // Element-wise activation kernel template.
135 template <typename scalar_t, scalar_t (*ACT_FN)(const scalar_t&)>
136 __global__ void activation_kernel(
csrc/ops.h
+3
@@ -48,6 +48,9 @@ void gelu_and_mul(torch::Tensor& out, torch::Tensor& input);
48
49 void gelu_tanh_and_mul(torch::Tensor& out, torch::Tensor& input);
50
51 +void fatrelu_and_mul(torch::Tensor& out, torch::Tensor& input,
52 + double threshold);
53 +
54 void gelu_new(torch::Tensor& out, torch::Tensor& input);
55
56 void gelu_fast(torch::Tensor& out, torch::Tensor& input);
csrc/torch_bindings.cpp
+4
@@ -60,6 +60,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
60 ops.def("gelu_tanh_and_mul(Tensor! out, Tensor input) -> ()");
61 ops.impl("gelu_tanh_and_mul", torch::kCUDA, &gelu_tanh_and_mul);
62
63 + // FATReLU implementation.
64 + ops.def("fatrelu_and_mul(Tensor! out, Tensor input, float threshold) -> ()");
65 + ops.impl("fatrelu_and_mul", torch::kCUDA, &fatrelu_and_mul);
66 +
67 // GELU implementation used in GPT-2.
68 ops.def("gelu_new(Tensor! out, Tensor input) -> ()");
69 ops.impl("gelu_new", torch::kCUDA, &gelu_new);
tests/kernels/test_activation.py
+16 -7
@@ -1,12 +1,13 @@
1 +import random
2 from typing import Type
3
4 import pytest
5 import torch
6
7 from tests.kernels.utils import opcheck
7 -from vllm.model_executor.layers.activation import (FastGELU, GeluAndMul,
8 - NewGELU, QuickGELU,
9 - SiluAndMul)
8 +from vllm.model_executor.layers.activation import (FastGELU, FatreluAndMul,
9 + GeluAndMul, NewGELU,
10 + QuickGELU, SiluAndMul)
11 from vllm.utils import seed_everything
12
13 from .allclose_default import get_default_atol, get_default_rtol
@@ -20,7 +21,8 @@ CUDA_DEVICES = [
21 ]
22
23
23 -@pytest.mark.parametrize("activation", ["silu", "gelu", "gelu_tanh"])
24 +@pytest.mark.parametrize("activation",
25 + ["silu", "gelu", "gelu_tanh", "fatrelu"])
26 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
27 @pytest.mark.parametrize("d", D)
28 @pytest.mark.parametrize("dtype", DTYPES)
@@ -47,16 +49,23 @@ def test_act_and_mul(
49 elif activation == "gelu_tanh":
50 layer = GeluAndMul(approximate="tanh")
51 fn = torch.ops._C.gelu_tanh_and_mul
52 + elif activation == "fatrelu":
53 + threshold = random.uniform(0, 1)
54 + layer = FatreluAndMul(threshold)
55 + fn = torch.ops._C.fatrelu_and_mul
56 out = layer(x)
57 ref_out = layer.forward_native(x)
52 - # The SiLU and GELU implementations are equivalent to the native PyTorch
53 - # implementations, so we can do exact comparison.
58 + # The SiLU, GELU and FatReLU implementations are equivalent to the native
59 + # PyTorch implementations, so we can do exact comparison.
60 torch.testing.assert_close(out, ref_out, atol=0.0, rtol=0.0)
61
62 d = x.shape[-1] // 2
63 output_shape = (x.shape[:-1] + (d, ))
64 out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
59 - opcheck(fn, (out, x))
65 + if activation == "fatrelu":
66 + opcheck(fn, (out, x, threshold))
67 + else:
68 + opcheck(fn, (out, x))
69
70
71 @pytest.mark.parametrize("activation", [(FastGELU, torch.ops._C.gelu_fast),
vllm/_custom_ops.py
+6
@@ -79,6 +79,12 @@ def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
79 torch.ops._C.gelu_tanh_and_mul(out, x)
80
81
82 +def fatrelu_and_mul(out: torch.Tensor,
83 + x: torch.Tensor,
84 + threshold: float = 0.0) -> None:
85 + torch.ops._C.fatrelu_and_mul(out, x, threshold)
86 +
87 +
88 def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None:
89 torch.ops._C.gelu_fast(out, x)
90
vllm/model_executor/layers/activation.py
+7 -1
@@ -39,7 +39,13 @@ class FatreluAndMul(CustomOp):
39 return x1 * x2
40
41 def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
42 - return self.forward_native(x)
42 + from vllm import _custom_ops as ops
43 +
44 + d = x.shape[-1] // 2
45 + output_shape = (x.shape[:-1] + (d, ))
46 + out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
47 + ops.fatrelu_and_mul(out, x, self.threshold)
48 + return out
49
50
51 @CustomOp.register("silu_and_mul")