@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d907be7dc7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[misc] remove python function call for custom activation op (#11885)

Co-authored-by: youkaichao <youkaichao@gmail.com>

cennn committed Jan 10, 2025 at 17:18 UTC d907be7dc7926e64d6240bf4425d7399eaed150e
2 files changed +46 -60
vllm/_custom_ops.py
-27
@@ -34,33 +34,6 @@ else:
34 from torch.library import impl_abstract as register_fake
35
36
37 -# activation ops
38 -def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
39 - torch.ops._C.gelu_and_mul(out, x)
40 -
41 -
42 -def gelu_tanh_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
43 - torch.ops._C.gelu_tanh_and_mul(out, x)
44 -
45 -
46 -def fatrelu_and_mul(out: torch.Tensor,
47 - x: torch.Tensor,
48 - threshold: float = 0.0) -> None:
49 - torch.ops._C.fatrelu_and_mul(out, x, threshold)
50 -
51 -
52 -def gelu_fast(out: torch.Tensor, x: torch.Tensor) -> None:
53 - torch.ops._C.gelu_fast(out, x)
54 -
55 -
56 -def gelu_new(out: torch.Tensor, x: torch.Tensor) -> None:
57 - torch.ops._C.gelu_new(out, x)
58 -
59 -
60 -def gelu_quick(out: torch.Tensor, x: torch.Tensor) -> None:
61 - torch.ops._C.gelu_quick(out, x)
62 -
63 -
37 # page attention ops
38 def paged_attention_v1(
39 out: torch.Tensor,
vllm/model_executor/layers/activation.py
+46 -33
@@ -30,6 +30,8 @@ class FatreluAndMul(CustomOp):
30 def __init__(self, threshold: float = 0.):
31 super().__init__()
32 self.threshold = threshold
33 + if current_platform.is_cuda_alike() or current_platform.is_cpu():
34 + self.op = torch.ops._C.fatrelu_and_mul
35
36 def forward_native(self, x: torch.Tensor) -> torch.Tensor:
37 d = x.shape[-1] // 2
@@ -39,12 +41,10 @@ class FatreluAndMul(CustomOp):
41 return x1 * x2
42
43 def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
42 - from vllm import _custom_ops as ops
43 -
44 d = x.shape[-1] // 2
45 output_shape = (x.shape[:-1] + (d, ))
46 out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
47 - ops.fatrelu_and_mul(out, x, self.threshold)
47 + self.op(out, x, self.threshold)
48 return out
49
50
@@ -103,6 +103,17 @@ class GeluAndMul(CustomOp):
103 self.approximate = approximate
104 if approximate not in ("none", "tanh"):
105 raise ValueError(f"Unknown approximate mode: {approximate}")
106 + if current_platform.is_cuda_alike() or current_platform.is_cpu():
107 + if approximate == "none":
108 + self.op = torch.ops._C.gelu_and_mul
109 + elif approximate == "tanh":
110 + self.op = torch.ops._C.gelu_tanh_and_mul
111 + elif current_platform.is_xpu():
112 + from vllm._ipex_ops import ipex_ops
113 + if approximate == "none":
114 + self.op = ipex_ops.gelu_and_mul
115 + else:
116 + self.op = ipex_ops.gelu_tanh_and_mul
117
118 def forward_native(self, x: torch.Tensor) -> torch.Tensor:
119 """PyTorch-native implementation equivalent to forward()."""
@@ -110,27 +121,17 @@ class GeluAndMul(CustomOp):
121 return F.gelu(x[..., :d], approximate=self.approximate) * x[..., d:]
122
123 def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
113 - from vllm import _custom_ops as ops
114 -
124 d = x.shape[-1] // 2
125 output_shape = (x.shape[:-1] + (d, ))
126 out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
118 - if self.approximate == "none":
119 - ops.gelu_and_mul(out, x)
120 - elif self.approximate == "tanh":
121 - ops.gelu_tanh_and_mul(out, x)
127 + self.op(out, x)
128 return out
129
130 def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
125 - from vllm._ipex_ops import ipex_ops as ops
126 -
131 d = x.shape[-1] // 2
132 output_shape = (x.shape[:-1] + (d, ))
133 out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
130 - if self.approximate == "none":
131 - ops.gelu_and_mul(out, x)
132 - elif self.approximate == "tanh":
133 - ops.gelu_tanh_and_mul(out, x)
134 + self.op(out, x)
135 return out
136
137 def extra_repr(self) -> str:
@@ -140,6 +141,14 @@ class GeluAndMul(CustomOp):
141 @CustomOp.register("gelu_new")
142 class NewGELU(CustomOp):
143
144 + def __init__(self):
145 + super().__init__()
146 + if current_platform.is_cuda_alike() or current_platform.is_cpu():
147 + self.op = torch.ops._C.gelu_new
148 + elif current_platform.is_xpu():
149 + from vllm._ipex_ops import ipex_ops
150 + self.op = ipex_ops.gelu_new
151 +
152 def forward_native(self, x: torch.Tensor) -> torch.Tensor:
153 """PyTorch-native implementation equivalent to forward()."""
154 c = math.sqrt(2.0 / math.pi)
@@ -147,58 +156,62 @@ class NewGELU(CustomOp):
156 (x + 0.044715 * torch.pow(x, 3.0))))
157
158 def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
150 - from vllm import _custom_ops as ops
151 -
159 out = torch.empty_like(x)
153 - ops.gelu_new(out, x)
160 + self.op(out, x)
161 return out
162
163 def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
157 - from vllm._ipex_ops import ipex_ops as ops
158 -
159 - return ops.gelu_new(x)
164 + return self.op(x)
165
166
167 @CustomOp.register("gelu_fast")
168 class FastGELU(CustomOp):
169
170 + def __init__(self):
171 + super().__init__()
172 + if current_platform.is_cuda_alike() or current_platform.is_cpu():
173 + self.op = torch.ops._C.gelu_fast
174 + elif current_platform.is_xpu():
175 + from vllm._ipex_ops import ipex_ops
176 + self.op = ipex_ops.gelu_fast
177 +
178 def forward_native(self, x: torch.Tensor) -> torch.Tensor:
179 """PyTorch-native implementation equivalent to forward()."""
180 return 0.5 * x * (1.0 + torch.tanh(x * 0.7978845608 *
181 (1.0 + 0.044715 * x * x)))
182
183 def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
171 - from vllm import _custom_ops as ops
172 -
184 out = torch.empty_like(x)
174 - ops.gelu_fast(out, x)
185 + self.op(out, x)
186 return out
187
188 def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
178 - from vllm._ipex_ops import ipex_ops as ops
179 -
180 - return ops.gelu_fast(x)
189 + return self.op(x)
190
191
192 @CustomOp.register("quick_gelu")
193 class QuickGELU(CustomOp):
194 # https://github.com/huggingface/transformers/blob/main/src/transformers/activations.py#L90
195 + def __init__(self):
196 + super().__init__()
197 + if current_platform.is_cuda_alike() or current_platform.is_cpu():
198 + self.op = torch.ops._C.gelu_quick
199 + elif current_platform.is_xpu():
200 + from vllm._ipex_ops import ipex_ops
201 + self.op = ipex_ops.gelu_quick
202 +
203 def forward_native(self, x: torch.Tensor) -> torch.Tensor:
204 """PyTorch-native implementation equivalent to forward()."""
205 return x * torch.sigmoid(1.702 * x)
206
207 def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
191 - from vllm import _custom_ops as ops
192 -
208 out = torch.empty_like(x)
194 - ops.gelu_quick(out, x)
209 + self.op(out, x)
210 return out
211
212 def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
198 - from vllm._ipex_ops import ipex_ops as ops
199 -
213 out = torch.empty_like(x)
201 - ops.gelu_quick(out, x)
214 + self.op(out, x)
215 return out
216
217 # TODO implement forward_xpu for QuickGELU