A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[platform] support pytorch custom op pluggable (#11328)
Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>
wangxiyuan committed
Jan 10, 2025 at 18:02 UTC
ef725feafcc1f2d6763cc888751fb2b36840587b
2 files changed
+11
vllm/model_executor/custom_op.py
+7
@@ -57,6 +57,11 @@ class CustomOp(nn.Module):
57
# PyTorch-native implementation.
58
return self.forward_native(*args, **kwargs)
59
60
+ def forward_oot(self, *args, **kwargs):
61
+ # By default, we assume that OOT ops are compatible with the
62
+ # PyTorch-native implementation.
63
+ return self.forward_native(*args, **kwargs)
64
+
65
def dispatch_forward(self):
66
# NOTE(woosuk): Here we assume that vLLM was built for only one
67
# specific backend. Currently, we do not support dynamic dispatching.
@@ -81,6 +86,8 @@ class CustomOp(nn.Module):
86
return self.forward_tpu
87
elif current_platform.is_xpu():
88
return self.forward_xpu
89
+ elif current_platform.is_out_of_tree():
90
+ return self.forward_oot
91
else:
92
return self.forward_cuda
93
vllm/platforms/interface.py
+4
@@ -45,6 +45,7 @@ class PlatformEnum(enum.Enum):
45
CPU = enum.auto()
46
NEURON = enum.auto()
47
OPENVINO = enum.auto()
48
+ OOT = enum.auto()
49
UNSPECIFIED = enum.auto()
50
51
@@ -107,6 +108,9 @@ class Platform:
108
def is_openvino(self) -> bool:
109
return self._enum == PlatformEnum.OPENVINO
110
111
+ def is_out_of_tree(self) -> bool:
112
+ return self._enum == PlatformEnum.OOT
113
+
114
def is_cuda_alike(self) -> bool:
115
"""Stateless version of :func:`torch.cuda.is_available`."""
116
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)