A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 01:15 UTC
[Quark] Fix MoE fp8 activation scale handling on mi300 (#34386)
Signed-off-by: Bowen Bao <bowenbao@amd.com>
Bowen Bao committed
Feb 19, 2026 at 21:27 UTC
d9e62c03eb98e3adcf82a2177f4a8b8f851406e4
1 file changed
+3
-3
vllm/model_executor/layers/quantization/quark/quark_moe.py
+3
-3
@@ -858,7 +858,7 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
858
layer.w2_input_scale = None
859
860
def process_weights_after_loading(self, layer):
861
- if self.static_input_scales:
861
+ if self.static_input_scales and self.input_dtype == "fp8":
862
# firstly, process activations if fp8 static input
863
if layer.w13_input_scale is None or layer.w2_input_scale is None:
864
raise ValueError(
@@ -883,14 +883,14 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
883
if current_platform.is_fp8_fnuz():
884
# Normalize the weights and scales
885
_, _, w13_input_scale = normalize_e4m3fn_to_e4m3fnuz(
886
- torch.empty_like(layer.w13_weight, dtype=torch.float8_e4m3fnuz),
886
+ torch.empty_like(layer.w13_weight, dtype=torch.float8_e4m3fn),
887
torch.empty_like(
888
layer.w13_weight_scale, dtype=layer.w13_weight_scale.dtype
889
),
890
layer.w13_input_scale,
891
)
892
_, _, w2_input_scale = normalize_e4m3fn_to_e4m3fnuz(
893
- torch.empty_like(layer.w2_weight, dtype=torch.float8_e4m3fnuz),
893
+ torch.empty_like(layer.w2_weight, dtype=torch.float8_e4m3fn),
894
torch.empty_like(
895
layer.w2_weight_scale, dtype=layer.w13_weight_scale.dtype
896
),