A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 14:45 UTC
[Bugfix][TPU] Return a Default fp8 MoE Backend (#32908)
Signed-off-by: Xiongfei Wei <isaacwxf23@gmail.com> Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
XiongfeiWei committed
Jan 26, 2026 at 15:46 UTC
510ed1e8d3b1da814d3f2b516008b8a0bc2ee464
1 file changed
+11
-3
vllm/model_executor/layers/fused_moe/oracle/fp8.py
+11
-3
@@ -35,6 +35,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils_fp8 import (
35
from vllm.model_executor.layers.quantization.utils.quant_utils import (
36
QuantKey,
37
)
38
+from vllm.platforms import current_platform
39
40
logger = init_logger(__name__)
41
@@ -330,9 +331,16 @@ def select_fp8_moe_backend(
331
else:
332
logger.debug_once(_make_log_unsupported(backend, reason), scope="local")
333
333
- raise NotImplementedError(
334
- "No FP8 MoE backend supports the deployment configuration."
335
- )
334
+ # TODO(rob): per discussion with TPU team, we need a way to register
335
+ # MoE backends by OOT plugins, rather than having an explicit list
336
+ # of AVAILBLE_BACKENDS. Enabling returning `Fp8MoeBackend.NONE` is
337
+ # a temporary measure until these register APIs are complete.
338
+ if current_platform.is_cuda() or current_platform.is_rocm():
339
+ raise NotImplementedError(
340
+ "No FP8 MoE backend supports the deployment configuration."
341
+ )
342
+
343
+ return Fp8MoeBackend.NONE, None
344
345
346
def convert_to_fp8_moe_kernel_format(