A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Bugfix] Fix broken CPU quantization due to triton import (#15038)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Mar 18, 2025 at 23:57 UTC
179a619c21c6656488b17690f54f04fcd2040578
1 file changed
+4
-1
vllm/model_executor/layers/quantization/gguf.py
+4
-1
@@ -10,7 +10,6 @@ from torch.nn.parameter import Parameter, UninitializedParameter
10
from vllm import _custom_ops as ops
11
from vllm.logger import init_logger
12
from vllm.model_executor.layers.activation import SiluAndMul
13
-from vllm.model_executor.layers.fused_moe.fused_moe import moe_align_block_size
13
from vllm.model_executor.layers.fused_moe.layer import (FusedMoE,
14
FusedMoEMethodBase)
15
from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
@@ -140,6 +139,10 @@ def _fused_moe_gguf(
139
qweight_type2: int,
140
act,
141
) -> torch.Tensor:
142
+ # lazy import to avoid triggering triton import in CPU backend
143
+ from vllm.model_executor.layers.fused_moe.fused_moe import (
144
+ moe_align_block_size)
145
+
146
out_hidden_states = torch.empty_like(x)
147
if qweight_type2 in MMQ_QUANT_TYPES and qweight_type in MMQ_QUANT_TYPES:
148
num_tokens, _ = x.shape