A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 20:15 UTC
[BugFix] add select_gemm_impl on CompressedTensorsWNA16MoEMethod to support LoRA (#31453)
Signed-off-by: JartX <sagformas@epdcenter.es>
JartX committed
Dec 30, 2025 at 20:20 UTC
07728bf5cd7165972f89e52e8b31ca28576262ec
1 file changed
+23
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+23
@@ -1996,6 +1996,29 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
1996
block_shape=[0, self.group_size],
1997
)
1998
1999
+ def select_gemm_impl(
2000
+ self,
2001
+ prepare_finalize: mk.FusedMoEPrepareAndFinalize,
2002
+ layer: torch.nn.Module,
2003
+ ) -> mk.FusedMoEPermuteExpertsUnpermute:
2004
+ if self.moe.is_lora_enabled:
2005
+ assert self.moe_quant_config is not None
2006
+ from vllm.triton_utils import HAS_TRITON
2007
+
2008
+ if HAS_TRITON:
2009
+ from vllm.model_executor.layers.fused_moe import TritonExperts
2010
+
2011
+ layer.w13_weight = layer.w13_weight_packed
2012
+ layer.w2_weight = layer.w2_weight_packed
2013
+ return TritonExperts(quant_config=self.moe_quant_config)
2014
+ else:
2015
+ raise NotImplementedError(
2016
+ "TritonExperts requires Triton. "
2017
+ "Install triton or disable LoRA for MoE."
2018
+ )
2019
+
2020
+ raise NotImplementedError
2021
+
2022
def apply(
2023
self,
2024
layer: FusedMoE,