A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 07:30 UTC
[Bugfix] Enable Triton MoE for FP8 per-tensor dynamic (#33300)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Jan 29, 2026 at 15:15 UTC
bfb9bdaf3f44fdcd4c6ff66b5e8c84834b092534
2 files changed
+3
vllm/model_executor/layers/fused_moe/fused_batched_moe.py
+1
@@ -933,6 +933,7 @@ class BatchedTritonExperts(mk.FusedMoEPermuteExpertsUnpermute):
933
SUPPORTED_W_A_FP8 = [
934
(kFp8Static128BlockSym, kFp8Dynamic128Sym),
935
(kFp8StaticChannelSym, kFp8DynamicTokenSym),
936
+ (kFp8StaticTensorSym, kFp8DynamicTokenSym),
937
(kFp8StaticTensorSym, kFp8StaticTensorSym),
938
(kFp8StaticTensorSym, kFp8DynamicTensorSym),
939
]
vllm/model_executor/layers/fused_moe/fused_moe.py
+2
@@ -45,6 +45,7 @@ from vllm.model_executor.layers.quantization.utils.ocp_mx_utils import OCP_MX_Sc
45
from vllm.model_executor.layers.quantization.utils.quant_utils import (
46
QuantKey,
47
kFp8Dynamic128Sym,
48
+ kFp8DynamicTensorSym,
49
kFp8DynamicTokenSym,
50
kFp8Static128BlockSym,
51
kFp8StaticChannelSym,
@@ -1932,6 +1933,7 @@ class TritonExperts(mk.FusedMoEPermuteExpertsUnpermute):
1933
(kFp8StaticChannelSym, kFp8DynamicTokenSym),
1934
(kFp8StaticTensorSym, kFp8DynamicTokenSym),
1935
(kFp8StaticTensorSym, kFp8StaticTensorSym),
1936
+ (kFp8StaticTensorSym, kFp8DynamicTensorSym),
1937
]
1938
return (weight_key, activation_key) in SUPPORTED_W_A
1939