A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
Fix CompressedTensorsWNA16MoE with grouped scales (#13769)
Michael Goin committed
Feb 25, 2025 at 03:17 UTC
4d251ad00ea11ecdf369aa10b9abecf96505b8dc
1 file changed
+2
-1
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+2
-1
@@ -527,7 +527,8 @@ class CompressedTensorsWNA16MoEMethod(CompressedTensorsMoEMethod):
527
replace_tensor("w13_weight_scale", marlin_w13_scales)
528
marlin_w2_scales = marlin_moe_permute_scales(
529
layer.w2_weight_scale,
530
- layer.w2_weight_scale.shape[1] * self.packed_factor,
530
+ layer.w2_weight_scale.shape[1] *
531
+ (self.group_size if self.group_size != -1 else self.packed_factor),
532
size_k2,
533
self.group_size,
534
self.num_bits,