A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[Bugfix] Lazy import fused_experts in BitsAndBytesMoEMethod to avoid break not-cuda-alike devices (#20822)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Jul 12, 2025 at 11:52 UTC
b1235c3e109746abcf638ed9bf76d43f03489b5e
1 file changed
+1
-1
vllm/model_executor/layers/quantization/bitsandbytes.py
+1
-1
@@ -5,7 +5,6 @@ from typing import Any, Callable, Optional, Union
5
6
import torch
7
8
-from vllm.model_executor.layers.fused_moe import fused_experts
8
from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,
9
FusedMoEMethodBase)
10
from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
@@ -467,6 +466,7 @@ class BitsAndBytesMoEMethod(FusedMoEMethodBase):
466
logical_to_physical_map: Optional[torch.Tensor] = None,
467
logical_replica_count: Optional[torch.Tensor] = None,
468
) -> torch.Tensor:
469
+ from vllm.model_executor.layers.fused_moe import fused_experts
470
471
if enable_eplb:
472
raise NotImplementedError(