A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 10:45 UTC
[Bugfix][ROCm] Fix MoRI + AITER FP8 dispatch compatibility for defer_input_quant (#37418)
Signed-off-by: Duyi-Wang <duyi.wang@amd.com>
Duyi-Wang committed
Mar 19, 2026 at 17:49 UTC
6a9cceb219fcbd6b1eb540ddfdc77ec160f0e209
2 files changed
+9
-7
vllm/model_executor/layers/fused_moe/mori_prepare_finalize.py
+3
-6
@@ -70,16 +70,13 @@ class MoriPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
70
- Optional dispatched expert topk IDs
71
- Optional dispatched expert topk weight
72
"""
73
- if defer_input_quant:
74
- raise NotImplementedError(
75
- f"{self.__class__.__name__} does not support defer_input_quant=True. "
76
- "Please select an MoE kernel that accepts quantized inputs."
77
- )
73
assert not apply_router_weight_on_input, (
74
"mori does not support apply_router_weight_on_input=True now."
75
)
76
scale = None
82
- if self.use_fp8_dispatch:
77
+ # When defer_input_quant is True, the expert kernel handles
78
+ # quantization internally, so skip FP8 dispatch quantization.
79
+ if self.use_fp8_dispatch and not defer_input_quant:
80
from aiter import QuantType, get_hip_quant
81
82
if quant_config.is_block_quantized:
vllm/model_executor/layers/fused_moe/rocm_aiter_fused_moe.py
+6
-1
@@ -295,7 +295,12 @@ def rocm_aiter_fused_experts(
295
class AiterExperts(mk.FusedMoEExpertsModular):
296
@property
297
def expects_unquantized_inputs(self) -> bool:
298
- return True
298
+ # When paired with MoRI, the prepare/finalize handles FP8
299
+ # quantization during dispatch to reduce network traffic,
300
+ # so we should not defer input quantization.
301
+ # Otherwise, AITER fused MoE kernels handle input quantization
302
+ # internally via a single fused kernel.
303
+ return not self.moe_config.use_mori_kernels
304
305
@staticmethod
306
def activation_format() -> mk.FusedMoEActivationFormat: