A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[misc] remove python function call for custom activation op (#11885)
Co-authored-by: youkaichao <youkaichao@gmail.com>
cennn committed
Jan 10, 2025 at 17:18 UTC
d907be7dc7926e64d6240bf4425d7399eaed150e
2 files changed
+46
-60
vllm/_custom_ops.py
-27
@@ -34,33 +34,6 @@ else:
34
from torch.library import impl_abstract as register_fake
35
36
37
-# activation ops
38
-def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
39
- torch.ops._C.gelu_and_mul(out, x)
40
-
41
-
42
-def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
43
- torch.ops._C.gelu_tanh_and_mul(out, x)
44
-
45
-
46
-def fatrelu_and_mul(out: torch.Tensor,
47
- x: torch.Tensor,
48
- threshold: float = 0.0) -> None:
49
- torch.ops._C.fatrelu_and_mul(out, x, threshold)
50
-
51
-
52
-def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None:
53
- torch.ops._C.gelu_fast(out, x)
54
-
55
-
56
-def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
57
- torch.ops._C.gelu_new(out, x)
58
-
59
-
60
-def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None:
61
- torch.ops._C.gelu_quick(out, x)
62
-
63
-
37
# page attention ops
38
def paged_attention_v1(
39
out: torch.Tensor,
vllm/model_executor/layers/activation.py
+46
-33
@@ -30,6 +30,8 @@ class FatreluAndMul(CustomOp):
30
def __init__(self, threshold: float = 0.):
31
super().__init__()
32
self.threshold = threshold
33
+ if current_platform.is_cuda_alike() or current_platform.is_cpu():
34
+ self.op = torch.ops._C.fatrelu_and_mul
35
36
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
37
d = x.shape[-1] // 2
@@ -39,12 +41,10 @@ class FatreluAndMul(CustomOp):
41
return x1 * x2
42
43
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
42
- from vllm import _custom_ops as ops
43
-
44
d = x.shape[-1] // 2
45
output_shape = (x.shape[:-1] + (d, ))
46
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
47
- ops.fatrelu_and_mul(out, x, self.threshold)
47
+ self.op(out, x, self.threshold)
48
return out
49
50
@@ -103,6 +103,17 @@ class GeluAndMul(CustomOp):
103
self.approximate = approximate
104
if approximate not in ("none", "tanh"):
105
raise ValueError(f"Unknown approximate mode: {approximate}")
106
+ if current_platform.is_cuda_alike() or current_platform.is_cpu():
107
+ if approximate == "none":
108
+ self.op = torch.ops._C.gelu_and_mul
109
+ elif approximate == "tanh":
110
+ self.op = torch.ops._C.gelu_tanh_and_mul
111
+ elif current_platform.is_xpu():
112
+ from vllm._ipex_ops import ipex_ops
113
+ if approximate == "none":
114
+ self.op = ipex_ops.gelu_and_mul
115
+ else:
116
+ self.op = ipex_ops.gelu_tanh_and_mul
117
118
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
119
"""PyTorch-native implementation equivalent to forward()."""
@@ -110,27 +121,17 @@ class GeluAndMul(CustomOp):
121
return F.gelu(x[..., :d], approximate=self.approximate) * x[..., d:]
122
123
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
113
- from vllm import _custom_ops as ops
114
-
124
d = x.shape[-1] // 2
125
output_shape = (x.shape[:-1] + (d, ))
126
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
118
- if self.approximate == "none":
119
- ops.gelu_and_mul(out, x)
120
- elif self.approximate == "tanh":
121
- ops.gelu_tanh_and_mul(out, x)
127
+ self.op(out, x)
128
return out
129
130
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
125
- from vllm._ipex_ops import ipex_ops as ops
126
-
131
d = x.shape[-1] // 2
132
output_shape = (x.shape[:-1] + (d, ))
133
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
130
- if self.approximate == "none":
131
- ops.gelu_and_mul(out, x)
132
- elif self.approximate == "tanh":
133
- ops.gelu_tanh_and_mul(out, x)
134
+ self.op(out, x)
135
return out
136
137
def extra_repr(self) -> str:
@@ -140,6 +141,14 @@ class GeluAndMul(CustomOp):
141
@CustomOp.register("gelu_new")
142
class NewGELU(CustomOp):
143
144
+ def __init__(self):
145
+ super().__init__()
146
+ if current_platform.is_cuda_alike() or current_platform.is_cpu():
147
+ self.op = torch.ops._C.gelu_new
148
+ elif current_platform.is_xpu():
149
+ from vllm._ipex_ops import ipex_ops
150
+ self.op = ipex_ops.gelu_new
151
+
152
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
153
"""PyTorch-native implementation equivalent to forward()."""
154
c = math.sqrt(2.0 / math.pi)
@@ -147,58 +156,62 @@ class NewGELU(CustomOp):
156
(x + 0.044715 * torch.pow(x, 3.0))))
157
158
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
150
- from vllm import _custom_ops as ops
151
-
159
out = torch.empty_like(x)
153
- ops.gelu_new(out, x)
160
+ self.op(out, x)
161
return out
162
163
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
157
- from vllm._ipex_ops import ipex_ops as ops
158
-
159
- return ops.gelu_new(x)
164
+ return self.op(x)
165
166
167
@CustomOp.register("gelu_fast")
168
class FastGELU(CustomOp):
169
170
+ def __init__(self):
171
+ super().__init__()
172
+ if current_platform.is_cuda_alike() or current_platform.is_cpu():
173
+ self.op = torch.ops._C.gelu_fast
174
+ elif current_platform.is_xpu():
175
+ from vllm._ipex_ops import ipex_ops
176
+ self.op = ipex_ops.gelu_fast
177
+
178
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
179
"""PyTorch-native implementation equivalent to forward()."""
180
return 0.5 * x * (1.0 + torch.tanh(x * 0.7978845608 *
181
(1.0 + 0.044715 * x * x)))
182
183
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
171
- from vllm import _custom_ops as ops
172
-
184
out = torch.empty_like(x)
174
- ops.gelu_fast(out, x)
185
+ self.op(out, x)
186
return out
187
188
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
178
- from vllm._ipex_ops import ipex_ops as ops
179
-
180
- return ops.gelu_fast(x)
189
+ return self.op(x)
190
191
192
@CustomOp.register("quick_gelu")
193
class QuickGELU(CustomOp):
194
# https://github.com/huggingface/transformers/blob/main/src/transformers/activations.py#L90
195
+ def __init__(self):
196
+ super().__init__()
197
+ if current_platform.is_cuda_alike() or current_platform.is_cpu():
198
+ self.op = torch.ops._C.gelu_quick
199
+ elif current_platform.is_xpu():
200
+ from vllm._ipex_ops import ipex_ops
201
+ self.op = ipex_ops.gelu_quick
202
+
203
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
204
"""PyTorch-native implementation equivalent to forward()."""
205
return x * torch.sigmoid(1.702 * x)
206
207
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
191
- from vllm import _custom_ops as ops
192
-
208
out = torch.empty_like(x)
194
- ops.gelu_quick(out, x)
209
+ self.op(out, x)
210
return out
211
212
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
198
- from vllm._ipex_ops import ipex_ops as ops
199
-
213
out = torch.empty_like(x)
201
- ops.gelu_quick(out, x)
214
+ self.op(out, x)
215
return out
216
217
# TODO implement forward_xpu for QuickGELU