A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[optimization] remove python function call for custom op (#11750)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 8, 2025 at 01:04 UTC
869579a702cb086cca6bd6ec4500f954a9adec1c
4 files changed
+15
-13
vllm/_custom_ops.py
-4
@@ -35,10 +35,6 @@ else:
35
36
37
# activation ops
38
-def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
39
- torch.ops._C.silu_and_mul(out, x)
40
-
41
-
38
def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
39
torch.ops._C.gelu_and_mul(out, x)
40
vllm/model_executor/layers/activation.py
+11
-6
@@ -10,6 +10,7 @@ from vllm.distributed import (divide, get_tensor_model_parallel_rank,
10
get_tensor_model_parallel_world_size)
11
from vllm.model_executor.custom_op import CustomOp
12
from vllm.model_executor.utils import set_weight_attrs
13
+from vllm.platforms import current_platform
14
from vllm.utils import LazyDict
15
16
@@ -58,27 +59,31 @@ class SiluAndMul(CustomOp):
59
return: (num_tokens, d) or (batch_size, seq_len, d)
60
"""
61
62
+ def __init__(self):
63
+ super().__init__()
64
+ if current_platform.is_cuda_alike():
65
+ self.op = torch.ops._C.silu_and_mul
66
+ elif current_platform.is_xpu():
67
+ import intel_extension_for_pytorch as ipex
68
+ self.op = ipex.llm.functional.silu_and_mul
69
+
70
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
71
"""PyTorch-native implementation equivalent to forward()."""
72
d = x.shape[-1] // 2
73
return F.silu(x[..., :d]) * x[..., d:]
74
75
def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
67
- from vllm import _custom_ops as ops
68
-
76
d = x.shape[-1] // 2
77
output_shape = (x.shape[:-1] + (d, ))
78
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
72
- ops.silu_and_mul(out, x)
79
+ self.op(out, x)
80
return out
81
82
def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
76
- from vllm._ipex_ops import ipex_ops as ops
77
-
83
d = x.shape[-1] // 2
84
output_shape = (x.shape[:-1] + (d, ))
85
out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
81
- ops.silu_and_mul(out, x)
86
+ self.op(out, x)
87
return out
88
89
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+2
-2
@@ -4,7 +4,6 @@ from typing import Optional
4
5
import torch
6
7
-from vllm import _custom_ops as ops
7
from vllm.model_executor.layers.fused_moe.fused_moe import (
8
fused_topk, moe_align_block_size, try_get_optimal_moe_config)
9
from vllm.scalar_type import scalar_types
@@ -301,7 +300,8 @@ def fused_marlin_moe(
300
False,
301
)
302
304
- ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, 2 * N))
303
+ torch.ops._C.silu_and_mul(intermediate_cache2,
304
+ intermediate_cache1.view(-1, 2 * N))
305
306
intermediate_cache3 = torch.ops._moe_C.marlin_gemm_moe(
307
intermediate_cache2,
vllm/model_executor/layers/fused_moe/fused_moe.py
+2
-1
@@ -753,7 +753,8 @@ def fused_experts_impl(hidden_states: torch.Tensor,
753
use_int8_w8a16=use_int8_w8a16,
754
block_shape=block_shape)
755
756
- ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
756
+ torch.ops._C.silu_and_mul(intermediate_cache2,
757
+ intermediate_cache1.view(-1, N))
758
759
invoke_fused_moe_kernel(intermediate_cache2,
760
w2,