A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 20:15 UTC
[BugFix] Fix LoRA Fp8 (#33879)
Signed-off-by: Daniel Serebrenik <daserebrenik@nvidia.com>
danisereb committed
Feb 5, 2026 at 19:25 UTC
5b2a9422f0f4cbdd69a9fed1dc1605838314ff81
1 file changed
+14
-8
vllm/lora/layers/fused_moe.py
+14
-8
@@ -130,14 +130,20 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
130
self.base_layer.ensure_moe_quant_config_init()
131
quant_config = self.base_layer.quant_method.moe_quant_config
132
133
- prepare_finalize = MoEPrepareAndFinalizeNoEP()
134
- m_fused_moe_fn = FusedMoEModularKernel(
135
- prepare_finalize,
136
- self.base_layer.quant_method.select_gemm_impl(
137
- prepare_finalize, self.base_layer
138
- ),
139
- self.base_layer.shared_experts,
140
- )
133
+ if getattr(self.base_layer.quant_method, "supports_internal_mk", False):
134
+ # Use the existing modular kernel from the quant method
135
+ m_fused_moe_fn = self.base_layer.quant_method.moe_mk
136
+ else:
137
+ # Create a new modular kernel via select_gemm_impl
138
+ prepare_finalize = MoEPrepareAndFinalizeNoEP()
139
+ m_fused_moe_fn = FusedMoEModularKernel(
140
+ prepare_finalize,
141
+ self.base_layer.quant_method.select_gemm_impl(
142
+ prepare_finalize, self.base_layer
143
+ ),
144
+ self.base_layer.shared_experts,
145
+ )
146
+
147
if quant_config.use_mxfp4_w4a16:
148
assert isinstance(
149
m_fused_moe_fn.fused_experts, (MarlinExperts, UnfusedOAITritonExperts)