A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Bugfix] Correct per_act_token in CompressedTensorsW8A8Fp8MoECutlassM… (#20937)
Signed-off-by: Ming Yang <minos.future@gmail.com>
Ming Yang committed
Jul 15, 2025 at 19:53 UTC
fcb9f879c1750774c03341c201ad8c1392d3ed23
1 file changed
+4
-6
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+4
-6
@@ -929,10 +929,8 @@ class CompressedTensorsW8A8Fp8MoECutlassMethod(CompressedTensorsMoEMethod):
929
scoring_func=scoring_func,
930
e_score_correction_bias=e_score_correction_bias)
931
932
- a1_scale = layer.w13_input_scale
933
- a2_scale = layer.w2_input_scale
934
- per_act_token = a1_scale.numel() != 1 if a1_scale is not None else (
935
- a2_scale.numel() != 1 if a2_scale is not None else False)
932
+ per_act_token = (
933
+ self.input_quant.strategy == QuantizationStrategy.TOKEN)
934
935
if self.fused_experts is None:
936
# If no modular kernel is provided, use cutlass_moe_fp8
@@ -950,8 +948,8 @@ class CompressedTensorsW8A8Fp8MoECutlassMethod(CompressedTensorsMoEMethod):
948
expert_map=None if self.disable_expert_map else expert_map,
949
w1_scale=layer.w13_weight_scale,
950
w2_scale=layer.w2_weight_scale,
953
- a1_scale=a1_scale,
954
- a2_scale=a2_scale,
951
+ a1_scale=layer.w13_input_scale,
952
+ a2_scale=layer.w2_input_scale,
953
)
954
else:
955
return self.fused_experts(