@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ef725feafc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[platform] support pytorch custom op pluggable (#11328)

Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com>

wangxiyuan committed Jan 10, 2025 at 18:02 UTC ef725feafcc1f2d6763cc888751fb2b36840587b
2 files changed +11
vllm/model_executor/custom_op.py
+7
@@ -57,6 +57,11 @@ class CustomOp(nn.Module):
57 # PyTorch-native implementation.
58 return self.forward_native(*args, **kwargs)
59
60 + def forward_oot(self, *args, **kwargs):
61 + # By default, we assume that OOT ops are compatible with the
62 + # PyTorch-native implementation.
63 + return self.forward_native(*args, **kwargs)
64 +
65 def dispatch_forward(self):
66 # NOTE(woosuk): Here we assume that vLLM was built for only one
67 # specific backend. Currently, we do not support dynamic dispatching.
@@ -81,6 +86,8 @@ class CustomOp(nn.Module):
86 return self.forward_tpu
87 elif current_platform.is_xpu():
88 return self.forward_xpu
89 + elif current_platform.is_out_of_tree():
90 + return self.forward_oot
91 else:
92 return self.forward_cuda
93
vllm/platforms/interface.py
+4
@@ -45,6 +45,7 @@ class PlatformEnum(enum.Enum):
45 CPU = enum.auto()
46 NEURON = enum.auto()
47 OPENVINO = enum.auto()
48 + OOT = enum.auto()
49 UNSPECIFIED = enum.auto()
50
51
@@ -107,6 +108,9 @@ class Platform:
108 def is_openvino(self) -> bool:
109 return self._enum == PlatformEnum.OPENVINO
110
111 + def is_out_of_tree(self) -> bool:
112 + return self._enum == PlatformEnum.OOT
113 +
114 def is_cuda_alike(self) -> bool:
115 """Stateless version of :func:`torch.cuda.is_available`."""
116 return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)