A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 21:00 UTC
[Bugfix] Fix gptq_marlin for deepseek-v3 (#13750)
Signed-off-by: dangshunya <dangshunya@baichuan-inc.com> Co-authored-by: dangshunya <dangshunya@baichuan-inc.com>
rainkert committed
Mar 5, 2025 at 12:25 UTC
0df25101d6dc3b83edeb813534b2d0931c79431a
1 file changed
+3
-1
vllm/model_executor/layers/quantization/gptq_marlin.py
+3
-1
@@ -569,7 +569,9 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
569
replace_parameter(layer, "w13_scales", marlin_w13_scales)
570
marlin_w2_scales = marlin_moe_permute_scales(
571
s=layer.w2_scales,
572
- size_k=layer.w2_scales.shape[1] * self.quant_config.pack_factor,
572
+ size_k=layer.w2_scales.shape[1] *
573
+ (self.quant_config.group_size if self.quant_config.group_size != -1
574
+ else self.quant_config.pack_factor),
575
size_n=layer.w2_scales.shape[2],
576
group_size=self.quant_config.group_size,
577
)