A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 00:15 UTC
[Bugfix] Add OOT backend option (#32471)
Signed-off-by: Iryna Boiko <iboiko@habana.ai>
Iryna Boiko committed
Jan 18, 2026 at 23:20 UTC
f5d17400303149bbb480f6abfb6f7bb646c1d895
1 file changed
+4
vllm/model_executor/layers/fused_moe/oracle/unquantized.py
+4
@@ -33,6 +33,7 @@ class UnquantizedMoeBackend(Enum):
33
CPU = "CPU"
34
XPU = "XPU"
35
TPU = "TPU"
36
+ OOT = "OOT"
37
38
39
# NOTE(zyongye): Unsupported backend means backend
@@ -42,6 +43,7 @@ UNSUPPORTED_BACKEND = [
43
UnquantizedMoeBackend.CPU,
44
UnquantizedMoeBackend.XPU,
45
UnquantizedMoeBackend.TPU,
46
+ UnquantizedMoeBackend.OOT,
47
]
48
49
@@ -95,6 +97,8 @@ def select_unquantized_moe_backend(
97
backend = UnquantizedMoeBackend.CPU
98
if current_platform.is_tpu():
99
backend = UnquantizedMoeBackend.TPU
100
+ if current_platform.is_out_of_tree():
101
+ backend = UnquantizedMoeBackend.OOT
102
103
logger.info_once(_make_log_backend(backend), scope="local")
104
return backend