A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[moe][quant] add weight name case for offset (#15515)
Signed-off-by: Mengqing Cao <cmq0113@163.com>
Mengqing Cao committed
Mar 27, 2025 at 12:50 UTC
fb22be5817cc772cd8bda02d73ca26bcac12751c
1 file changed
+3
-2
vllm/model_executor/layers/fused_moe/layer.py
+3
-2
@@ -699,8 +699,9 @@ class FusedMoE(torch.nn.Module):
699
tp_rank=self.tp_rank)
700
return
701
702
- # Case weight scales and zero_points
703
- if ("scale" in weight_name or "zero" in weight_name):
702
+ # Case weight scales, zero_points and offset
703
+ if ("scale" in weight_name or "zero" in weight_name
704
+ or "offset" in weight_name):
705
# load the weight scales and zp based on the quantization scheme
706
# supported weight scales/zp can be found in
707
# FusedMoeWeightScaleSupported