A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 11:15 UTC
[ROCm][Bugfix]: W4A4 MOE using emulation instead of AITER on MXFP4-supported hardware (#41175)
Signed-off-by: Rohan138 <rohanpotdar138@gmail.com>
Rohan Potdar committed
Apr 29, 2026 at 16:39 UTC
944e138bcf39e9236bbfd49d98f00fb45e6cea54
2 files changed
+4
-7
vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py
+2
-1
@@ -252,7 +252,8 @@ def rocm_aiter_fused_experts(
252
253
else:
254
quant_method = QuantMethod.NO.value
255
- # mxfp4: both w4a4 (quark) and w4a16 (oracle CK) use BLOCK_1X32
255
+ # mxfp4 i.e. w4a4, w4a16 uses BLOCK_1X32
256
+ # mxfp6 and mxfp8 are unsupported in AITER currently and use emulation instead
257
if quant_config.use_mxfp4_w4a4 or quant_config.use_mxfp4_w4a16:
258
quant_method = QuantMethod.BLOCK_1X32.value
259
# w8a8 block-scaled
vllm/model_executor/layers/quantization/quark/quark_moe.py
+2
-6
@@ -1025,12 +1025,8 @@ class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
1025
get_current_vllm_config().model_config.hf_config, "model_type", None
1026
)
1027
1028
- # TODO(aiter): extend once rocm_aiter_fused_experts gains dispatch
1029
- # for the other OCP MX schemes. Today its CK MoE kernel only has an
1030
- # entry for `w_mxfp4` (w4a16); mixed schemes like `w_mxfp4_a_mxfp6_*`
1031
- # fall through to QuantMethod.NO and raise "Unsupported kernel config
1032
- # for moe heuristic dispatch".
1033
- _AITER_NATIVE_OCP_MX_SCHEMES = ("w_mxfp4",)
1028
+ # TODO: Remove once all OCP MX schemes use the kernel abstraction
1029
+ _AITER_NATIVE_OCP_MX_SCHEMES = ("w_mxfp4", "w_mxfp4_a_mxfp4")
1030
self.emulate = (
1031
not current_platform.supports_mx()
1032
or self.ocp_mx_scheme not in _AITER_NATIVE_OCP_MX_SCHEMES