A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 05:00 UTC
[Bug] Fix compressed tensor not using deepgemm (#30820)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
Wentao Ye committed
Dec 18, 2025 at 14:45 UTC
97000a2be7e318be1a3eb172f9abf2d67dbe73bf
2 files changed
+10
-1
vllm/model_executor/layers/fused_moe/fused_moe.py
-1
@@ -1696,7 +1696,6 @@ def fused_experts(
1696
and (is_deep_gemm_e8m0_used() or _valid_deep_gemm(hidden_states, w1, w2))
1697
):
1698
assert quant_config is not None
1699
- assert apply_router_weight_on_input is False
1699
return deep_gemm_moe_fp8(
1700
hidden_states=hidden_states,
1701
w1=w1,
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe.py
+10
@@ -96,6 +96,7 @@ from vllm.utils.deep_gemm import (
96
get_col_major_tma_aligned_tensor,
97
get_mk_alignment_for_contiguous_layout,
98
is_deep_gemm_e8m0_used,
99
+ is_deep_gemm_supported,
100
)
101
from vllm.utils.import_utils import has_deep_gemm
102
@@ -716,6 +717,13 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
717
get_marlin_input_dtype(layer_name) if self.use_marlin else None
718
)
719
720
+ self.allow_deep_gemm = (
721
+ self.block_quant
722
+ and envs.VLLM_MOE_USE_DEEP_GEMM
723
+ and is_deep_gemm_supported()
724
+ and list(self.weight_block_size) == get_mk_alignment_for_contiguous_layout()
725
+ )
726
+
727
def create_weights(
728
self,
729
layer: torch.nn.Module,
@@ -1231,6 +1239,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
1239
if self.disable_expert_map
1240
else layer.expert_map, # ???
1241
quant_config=self.moe_quant_config,
1242
+ allow_deep_gemm=self.allow_deep_gemm,
1243
)
1244
else:
1245
from vllm.model_executor.layers.fused_moe.cutlass_moe import (
@@ -1272,6 +1281,7 @@ class CompressedTensorsW8A8Fp8MoEMethod(CompressedTensorsMoEMethod):
1281
global_num_experts=layer.global_num_experts,
1282
expert_map=layer.expert_map,
1283
quant_config=self.moe_quant_config,
1284
+ allow_deep_gemm=self.allow_deep_gemm,
1285
)
1286
1287
@property