A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Bugfix] Fix missing per_act_token parameter in compressed_tensors_moe (#20509)
Signed-off-by: Lu Fang <fanglu@fb.com>
Lucia Fang committed
Jul 6, 2025 at 12:08 UTC
432870829d5143840c45296b8c1f34e5f561fa85
1 file changed
+4
-1
vllm/model_executor/layers/fused_moe/cutlass_moe.py
+4
-1
@@ -322,7 +322,7 @@ def cutlass_moe_fp8(
322
topk_ids: torch.Tensor,
323
w1_scale: torch.Tensor,
324
w2_scale: torch.Tensor,
325
- per_act_token: bool,
325
+ per_act_token: Optional[bool] = None,
326
activation: str = "silu",
327
a1_scale: Optional[torch.Tensor] = None,
328
a2_scale: Optional[torch.Tensor] = None,
@@ -366,6 +366,9 @@ def cutlass_moe_fp8(
366
Returns:
367
- torch.Tensor: The fp16 output tensor after applying the MoE layer.
368
"""
369
+ if per_act_token is None:
370
+ per_act_token = a1_scale.numel() != 1 if a1_scale is not None else (
371
+ a2_scale.numel() != 1 if a2_scale is not None else False)
372
per_out_ch = w1_scale.numel() != w1_q.size(0)
373
374
num_experts = global_num_experts if global_num_experts != -1 else w1_q.size(