@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bd620b01fb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Kernel][CPU] Add Quick `gelu` to CPU (#5717)

Roger Wang committed Jun 20, 2024 at 23:39 UTC bd620b01fb74d5269ca6fc0fd32f66bfb205a358
4 files changed +29
csrc/cpu/activation.cpp
+19
@@ -59,6 +59,13 @@ FORCE_INLINE vec_op::FP32Vec8 gelu_fast_act(const vec_op::FP32Vec8& x) {
59 return w3 * x * (ones + t);
60 }
61
62 +FORCE_INLINE vec_op::FP32Vec8 gelu_quick_act(const vec_op::FP32Vec8& x) {
63 + const vec_op::FP32Vec8 zeros(0.0);
64 + const vec_op::FP32Vec8 ones(1.0);
65 + const vec_op::FP32Vec8 w1(1.702f);
66 + return x / (ones + (zeros - w1 * x).exp());
67 +}
68 +
69 FORCE_INLINE vec_op::FP32Vec8 gelu_act(const vec_op::FP32Vec8& x) {
70 const vec_op::FP32Vec8 ones(1.0);
71 const vec_op::FP32Vec8 w1(M_SQRT1_2);
@@ -142,3 +149,15 @@ void gelu_fast(torch::Tensor& out, torch::Tensor& input) {
149 CPU_KERNEL_GUARD_OUT(gelu_fast_impl)
150 });
151 }
152 +
153 +void gelu_quick(torch::Tensor& out, torch::Tensor& input) {
154 + int num_tokens = input.numel() / input.size(-1);
155 + int d = input.size(-1);
156 +
157 + VLLM_DISPATCH_FLOATING_TYPES(input.scalar_type(), "gelu_quick_impl", [&] {
158 + CPU_KERNEL_GUARD_IN(gelu_quick_impl)
159 + activation_kernel<scalar_t, gelu_quick_act, false>(
160 + num_tokens, d, input.data_ptr<scalar_t>(), out.data_ptr<scalar_t>());
161 + CPU_KERNEL_GUARD_OUT(gelu_quick_impl)
162 + });
163 +}
csrc/cpu/torch_bindings.cpp
+4
@@ -58,6 +58,10 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
58 ops.def("gelu_fast(Tensor! out, Tensor input) -> ()");
59 ops.impl("gelu_fast", torch::kCPU, &gelu_fast);
60
61 + // Quick GELU implementation.
62 + ops.def("gelu_quick(Tensor! out, Tensor input) -> ()");
63 + ops.impl("gelu_quick", torch::kCPU, &gelu_quick);
64 +
65 // Layernorm
66 // Apply Root Mean Square (RMS) Normalization to the input tensor.
67 ops.def(
vllm/_ipex_ops.py
+3
@@ -43,6 +43,9 @@ class ipex_ops:
43 def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
44 out.copy_(torch.nn.functional.gelu(x))
45
46 + # TODO add implementation of gelu_quick here
47 + # def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None:
48 +
49 def paged_attention_v1(
50 out: torch.Tensor,
51 query: torch.Tensor,
vllm/model_executor/layers/activation.py
+3
@@ -155,6 +155,9 @@ class QuickGELU(CustomOp):
155 ops.gelu_quick(out, x)
156 return out
157
158 + # TODO implement forward_xpu for QuickGELU
159 + # def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
160 +
161
162 class ScaledActivation(nn.Module):
163 """An activation function with post-scale parameters.