A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 01:00 UTC
[Bugfix] Fix FP8 MoE LoRA (#29890)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 5, 2025 at 02:17 UTC
652ba93da36d793e7f3ff8a0ecdb5d6b00107e68
1 file changed
+9
-3
vllm/model_executor/layers/quantization/fp8.py
+9
-3
@@ -124,12 +124,16 @@ class Fp8MoeBackend(Enum):
124
125
126
def get_fp8_moe_backend(
127
- block_quant: bool, moe_parallel_config: FusedMoEParallelConfig
127
+ block_quant: bool,
128
+ moe_parallel_config: FusedMoEParallelConfig,
129
+ with_lora_support: bool,
130
) -> Fp8MoeBackend:
131
"""
132
Select the primary FP8 MoE backend
133
Note: Shape-specific fallbacks may still occur at runtime.
134
"""
135
+ if with_lora_support:
136
+ return Fp8MoeBackend.TRITON
137
# Prefer FlashInfer backends on supported GPUs; allow SM90 and SM100.
138
if (
139
current_platform.is_cuda()
@@ -665,7 +669,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
669
self.weight_block_size = self.quant_config.weight_block_size
670
self.block_quant: bool = self.weight_block_size is not None
671
self.fp8_backend = get_fp8_moe_backend(
668
- self.block_quant, layer.moe_parallel_config
672
+ self.block_quant, layer.moe_parallel_config, self.moe.is_lora_enabled
673
)
674
675
self.marlin_input_dtype = None
@@ -1084,6 +1088,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
1088
from vllm.model_executor.layers.fused_moe import (
1089
BatchedDeepGemmExperts,
1090
BatchedTritonExperts,
1091
+ TritonExperts,
1092
TritonOrDeepGemmExperts,
1093
)
1094
@@ -1116,7 +1121,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
1121
num_dispatchers=prepare_finalize.num_dispatchers(),
1122
quant_config=self.moe_quant_config,
1123
)
1119
-
1124
+ elif self.moe.is_lora_enabled:
1125
+ return TritonExperts(quant_config=self.moe_quant_config)
1126
elif self.flashinfer_moe_backend == FlashinferMoeBackend.CUTLASS:
1127
# Select GEMM experts with block-scale when weights are block-quantized
1128
experts = select_cutlass_fp8_gemm_impl(