A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] [AITER] [ROCm] Fix Quark MoE Quant Config and AITER Fused MoE quant type logic (#27029)
Signed-off-by: vllmellm <vllm.ellm@embeddedllm.com>
vllmellm committed
Oct 18, 2025 at 02:51 UTC
e33ee23ee3cde5aa69912d9d03aa31421851662c
3 files changed
+14
-3
vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py
+2
@@ -492,6 +492,8 @@ def rocm_aiter_fused_experts(
492
assert quant_config.w1_scale is not None
493
assert quant_config.w2_scale is not None
494
quant_method = QuantMethod.BLOCK_128x128.value
495
+ elif quant_config.use_fp8_w8a8 and quant_config.per_out_ch_quant:
496
+ quant_method = QuantMethod.PER_TOKEN.value
497
elif quant_config.use_fp8_w8a8:
498
# Currently only per tensor quantization method is enabled.
499
quant_method = QuantMethod.PER_TENSOR.value
vllm/model_executor/layers/quantization/quark/quark_moe.py
+2
-1
@@ -342,7 +342,8 @@ class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
342
w2_scale=layer.w2_weight_scale,
343
a1_scale=layer.w13_input_scale,
344
a2_scale=layer.w2_input_scale,
345
- per_act_token_quant=self.weight_qscheme == "per_channel",
345
+ per_act_token_quant=self.input_qscheme == "per_channel",
346
+ per_out_ch_quant=self.weight_qscheme == "per_channel",
347
)
348
349
def apply(
vllm/model_executor/layers/quantization/utils/w8a8_utils.py
+10
-2
@@ -464,8 +464,16 @@ class Fp8LinearOp:
464
else:
465
qinput, x_scale = input_2d, input_scale
466
467
- per_tensor_weights = weight_scale.numel() == 1
468
- per_tensor_activations = x_scale.numel() == 1
467
+ # Must have dim() conditions
468
+ # In per-token quant scenario, when the number of token is 1,
469
+ # the scale will only have 1 elements.
470
+ # Without checking the dim(),
471
+ # we cannot distingushes between per-tensor and per-token quant.
472
+ # Example:
473
+ # When the number of token is 1, per-token scale is [[1]]
474
+ # When per-tensor scale is [1] or ().
475
+ per_tensor_weights = (weight_scale.numel() == 1) and weight_scale.dim() < 2
476
+ per_tensor_activations = (x_scale.numel() == 1) and x_scale.dim() < 2
477
478
# TODO(luka) do this dispatch during init (after ScaledMM refactor)
479
w8a8_scaled_mm_func = dispatch_w8a8_scaled_mm(