@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 869579a702
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[optimization] remove python function call for custom op (#11750)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 8, 2025 at 01:04 UTC 869579a702cb086cca6bd6ec4500f954a9adec1c
4 files changed +15 -13
vllm/_custom_ops.py
-4
@@ -35,10 +35,6 @@ else:
35
36
37 # activation ops
38 -def silu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
39 - torch.ops._C.silu_and_mul(out, x)
40 -
41 -
38 def gelu_and_mul(out: torch.Tensor, x: torch.Tensor) -> None:
39 torch.ops._C.gelu_and_mul(out, x)
40
vllm/model_executor/layers/activation.py
+11 -6
@@ -10,6 +10,7 @@ from vllm.distributed import (divide, get_tensor_model_parallel_rank,
10 get_tensor_model_parallel_world_size)
11 from vllm.model_executor.custom_op import CustomOp
12 from vllm.model_executor.utils import set_weight_attrs
13 +from vllm.platforms import current_platform
14 from vllm.utils import LazyDict
15
16
@@ -58,27 +59,31 @@ class SiluAndMul(CustomOp):
59 return: (num_tokens, d) or (batch_size, seq_len, d)
60 """
61
62 + def __init__(self):
63 + super().__init__()
64 + if current_platform.is_cuda_alike():
65 + self.op = torch.ops._C.silu_and_mul
66 + elif current_platform.is_xpu():
67 + import intel_extension_for_pytorch as ipex
68 + self.op = ipex.llm.functional.silu_and_mul
69 +
70 def forward_native(self, x: torch.Tensor) -> torch.Tensor:
71 """PyTorch-native implementation equivalent to forward()."""
72 d = x.shape[-1] // 2
73 return F.silu(x[..., :d]) * x[..., d:]
74
75 def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
67 - from vllm import _custom_ops as ops
68 -
76 d = x.shape[-1] // 2
77 output_shape = (x.shape[:-1] + (d, ))
78 out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
72 - ops.silu_and_mul(out, x)
79 + self.op(out, x)
80 return out
81
82 def forward_xpu(self, x: torch.Tensor) -> torch.Tensor:
76 - from vllm._ipex_ops import ipex_ops as ops
77 -
83 d = x.shape[-1] // 2
84 output_shape = (x.shape[:-1] + (d, ))
85 out = torch.empty(output_shape, dtype=x.dtype, device=x.device)
81 - ops.silu_and_mul(out, x)
86 + self.op(out, x)
87 return out
88
89
vllm/model_executor/layers/fused_moe/fused_marlin_moe.py
+2 -2
@@ -4,7 +4,6 @@ from typing import Optional
4
5 import torch
6
7 -from vllm import _custom_ops as ops
7 from vllm.model_executor.layers.fused_moe.fused_moe import (
8 fused_topk, moe_align_block_size, try_get_optimal_moe_config)
9 from vllm.scalar_type import scalar_types
@@ -301,7 +300,8 @@ def fused_marlin_moe(
300 False,
301 )
302
304 - ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, 2 * N))
303 + torch.ops._C.silu_and_mul(intermediate_cache2,
304 + intermediate_cache1.view(-1, 2 * N))
305
306 intermediate_cache3 = torch.ops._moe_C.marlin_gemm_moe(
307 intermediate_cache2,
vllm/model_executor/layers/fused_moe/fused_moe.py
+2 -1
@@ -753,7 +753,8 @@ def fused_experts_impl(hidden_states: torch.Tensor,
753 use_int8_w8a16=use_int8_w8a16,
754 block_shape=block_shape)
755
756 - ops.silu_and_mul(intermediate_cache2, intermediate_cache1.view(-1, N))
756 + torch.ops._C.silu_and_mul(intermediate_cache2,
757 + intermediate_cache1.view(-1, N))
758
759 invoke_fused_moe_kernel(intermediate_cache2,
760 w2,