A high-throughput and memory-efficient inference and serving engine for LLMs
[ROCm][Quant] Minimax-M3: Enable fp8_per_channel for bf16 weights on mi300x (#45854)
Signed-off-by: Hongxia Yang <hongxia.yang@amd.com> Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com> Co-authored-by: tjtanaa <tunjian.tan@embeddedllm.com>
Hongxia Yang committed
Jun 17, 2026 at 08:02 UTC
e28e8c87820d620f3983ef6a6ef6fdd67deb7936
5 files changed
+11
vllm/model_executor/layers/fused_moe/config.py
+4
@@ -603,6 +603,8 @@ def fp8_w8a8_moe_quant_config(
603
a2_gscale: torch.Tensor | None = None,
604
g1_alphas: torch.Tensor | None = None,
605
g2_alphas: torch.Tensor | None = None,
606
+ gemm1_alpha: float | None = None,
607
+ gemm1_beta: float | None = None,
608
gemm1_clamp_limit: float | None = None,
609
) -> FusedMoEQuantConfig:
610
"""
@@ -623,6 +625,8 @@ def fp8_w8a8_moe_quant_config(
625
per_act_token_quant=per_act_token_quant,
626
per_out_ch_quant=per_out_ch_quant,
627
block_shape=block_shape,
628
+ gemm1_alpha=gemm1_alpha,
629
+ gemm1_beta=gemm1_beta,
630
gemm1_clamp_limit=gemm1_clamp_limit,
631
)
632
vllm/model_executor/layers/fused_moe/oracle/fp8.py
+2
@@ -568,6 +568,8 @@ def make_fp8_moe_quant_config(
568
block_shape=block_shape,
569
per_act_token_quant=per_act_token_quant,
570
per_out_ch_quant=per_out_ch_quant,
571
+ gemm1_alpha=gemm1_alpha,
572
+ gemm1_beta=gemm1_beta,
573
gemm1_clamp_limit=swiglu_limit,
574
)
575
vllm/model_executor/layers/quantization/fp8.py
+2
@@ -787,6 +787,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
787
a2_scale=a2_scale,
788
block_shape=self.weight_block_size,
789
swiglu_limit=getattr(layer, "swiglu_limit", None),
790
+ gemm1_alpha=getattr(layer, "swiglu_alpha", None),
791
+ gemm1_beta=getattr(layer, "swiglu_beta", None),
792
)
793
794
# Inject biases into the quant config if the model has them
vllm/model_executor/layers/quantization/online/fp8.py
+2
@@ -483,6 +483,8 @@ class _Fp8OnlineMoEBase(OnlineMoEMethodBase):
483
per_act_token_quant=self.per_act_token_quant,
484
per_out_ch_quant=self.per_out_ch_quant,
485
swiglu_limit=getattr(layer, "swiglu_limit", None),
486
+ gemm1_alpha=getattr(layer, "swiglu_alpha", None),
487
+ gemm1_beta=getattr(layer, "swiglu_beta", None),
488
)
489
490
vllm/platforms/rocm.py
+1
@@ -459,6 +459,7 @@ class RocmPlatform(Platform):
459
"modelopt_mixed",
460
"fp8_per_tensor",
461
"fp8_per_block",
462
+ "fp8_per_channel",
463
"online",
464
"gpt_oss_mxfp4",
465
]