A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Kernel][CPU] Add Quick `gelu` to CPU (#5717)
Roger Wang committed
Jun 20, 2024 at 23:39 UTC
bd620b01fb74d5269ca6fc0fd32f66bfb205a358
4 files changed
+29
csrc/cpu/activation.cpp
+19
@@ -59,6 +59,13 @@ FORCE_INLINE vec_op::FP32Vec8 gelu_fast_act(const vec_op::FP32Vec8& x) {
59
return w3 * x * (ones + t);
60
}
61
62
+FORCE_INLINE vec_op::FP32Vec8 gelu_quick_act(const vec_op::FP32Vec8& x) {
63
+ const vec_op::FP32Vec8 zeros(0.0);
64
+ const vec_op::FP32Vec8 ones(1.0);
65
+ const vec_op::FP32Vec8 w1(1.702f);
66
+ return x / (ones + (zeros - w1 * x).exp());
67
+}
68
+
69
FORCE_INLINE vec_op::FP32Vec8 gelu_act(const vec_op::FP32Vec8& x) {
70
const vec_op::FP32Vec8 ones(1.0);
71
const vec_op::FP32Vec8 w1(M_SQRT1_2);
@@ -142,3 +149,15 @@ void gelu_fast(torch::Tensor& out, torch::Tensor& input) {
149
CPU_KERNEL_GUARD_OUT(gelu_fast_impl)
150
});
151
}
152
+
153
+void gelu_quick(torch::Tensor& out, torch::Tensor& input) {
154
+ int num_tokens = input.numel() / input.size(-1);
155
+ int d = input.size(-1);
156
+
157
+ VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_quick_impl", [&] {
158
+ CPU_KERNEL_GUARD_IN(gelu_quick_impl)
159
+ activation_kernel<scalar_t, gelu_quick_act, false>(
160
+ num_tokens, d, input.data_ptr<scalar_t>(), out.data_ptr<scalar_t>());
161
+ CPU_KERNEL_GUARD_OUT(gelu_quick_impl)
162
+ });
163
+}
csrc/cpu/torch_bindings.cpp
+4
@@ -58,6 +58,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
58
ops.def("gelu_fast(Tensor! out, Tensor input) -> ()");
59
ops.impl("gelu_fast", torch::kCPU, &gelu_fast);
60
61
+ // Quick GELU implementation.
62
+ ops.def("gelu_quick(Tensor! out, Tensor input) -> ()");
63
+ ops.impl("gelu_quick", torch::kCPU, &gelu_quick);
64
+
65
// Layernorm
66
// Apply Root Mean Square (RMS) Normalization to the input tensor.
67
ops.def(
vllm/_ipex_ops.py
+3
@@ -43,6 +43,9 @@ class ipex_ops:
43
def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
44
out.copy_(torch.nn.functional.gelu(x))
45
46
+ # TODO add implementation of gelu_quick here
47
+ # def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None:
48
+
49
def paged_attention_v1(
50
out: torch.Tensor,
51
query: torch.Tensor,
vllm/model_executor/layers/activation.py
+3
@@ -155,6 +155,9 @@ class QuickGELU(CustomOp):
155
ops.gelu_quick(out, x)
156
return out
157
158
+ # TODO implement forward_xpu for QuickGELU
159
+ # def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
160
+
161
162
class ScaledActivation(nn.Module):
163
"""An activation function with post-scale parameters.