A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 03:00 UTC
[Bugfix] Fix DSV4 Base model swiglu limit issue in FP8 path (#42855)
Signed-off-by: Chengze Fan <chengze@meta.com> Signed-off-by: Chengze Fan <fancz2002@gmail.com> Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
Chengze Fan committed
May 21, 2026 at 19:43 UTC
2998a047aad7d48bf0399f19b36f1a4d749c59c2
3 files changed
+38
-4
vllm/model_executor/layers/fused_moe/experts/flashinfer_cutlass_moe.py
+16
-4
@@ -97,6 +97,13 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
97
self.max_capture_size = (
98
get_current_vllm_config().compilation_config.max_cudagraph_capture_size
99
)
100
+ self.gemm1_clamp_limit: torch.Tensor | None = None
101
+ if quant_config.gemm1_clamp_limit is not None:
102
+ self.gemm1_clamp_limit = torch.tensor(
103
+ [quant_config.gemm1_clamp_limit] * self.num_experts,
104
+ dtype=torch.float32,
105
+ device=self.device,
106
+ )
107
108
if quant_config.weight_quant_dtype == "mxfp4":
109
# This value is used specifically for gpt-oss,
@@ -107,9 +114,12 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
114
self.gemm1_beta = torch.tensor(
115
[1.0] * self.num_experts, dtype=torch.float32, device=self.device
116
)
110
- self.gemm1_clamp_limit = torch.tensor(
111
- [7.0] * self.num_experts, dtype=torch.float32, device=self.device
112
- )
117
+ if self.gemm1_clamp_limit is None:
118
+ self.gemm1_clamp_limit = torch.tensor(
119
+ [7.0] * self.num_experts,
120
+ dtype=torch.float32,
121
+ device=self.device,
122
+ )
123
if quant_config.quant_dtype == "mxfp8":
124
self.fake_input_scale = torch.ones(
125
self.num_experts,
@@ -277,7 +287,9 @@ class FlashInferExperts(mk.FusedMoEExpertsModular):
287
fc2_expert_biases = None
288
swiglu_alpha = None
289
swiglu_beta = None
280
- swiglu_limit = None
290
+ swiglu_limit = (
291
+ self.gemm1_clamp_limit if activation == MoEActivation.SILU else None
292
+ )
293
use_mxfp8_act_scaling = False
294
use_w4_group_scaling = False
295
# Select quantization metadata based on FP8 format/path
vllm/model_executor/layers/fused_moe/experts/fused_batched_moe.py
+11
@@ -19,6 +19,7 @@ from vllm.model_executor.layers.fused_moe.utils import (
19
_resize_cache,
20
moe_kernel_quantize_input,
21
normalize_batched_scales_shape,
22
+ swiglu_limit_func,
23
)
24
from vllm.model_executor.layers.quantization.utils.quant_utils import (
25
QuantKey,
@@ -805,6 +806,16 @@ class BatchedTritonExperts(mk.FusedMoEExpertsModular):
806
# Let PrepareAndFinalize::finalize() decide the impl.
807
return TopKWeightAndReduceDelegate()
808
809
+ def activation(
810
+ self, activation: MoEActivation, output: torch.Tensor, input: torch.Tensor
811
+ ) -> None:
812
+ gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
813
+ if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
814
+ swiglu_limit_func(output, input, float(gemm1_clamp_limit))
815
+ return
816
+
817
+ super().activation(activation, output, input)
818
+
819
def workspace_shapes(
820
self,
821
M: int,
vllm/model_executor/layers/fused_moe/experts/triton_moe.py
+11
@@ -30,6 +30,7 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
30
from vllm.model_executor.layers.fused_moe.utils import (
31
_resize_cache,
32
moe_kernel_quantize_input,
33
+ swiglu_limit_func,
34
)
35
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
36
is_deep_gemm_e8m0_used,
@@ -129,6 +130,16 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
130
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
131
return TopKWeightAndReduceNoOP()
132
133
+ def activation(
134
+ self, activation: MoEActivation, output: torch.Tensor, input: torch.Tensor
135
+ ) -> None:
136
+ gemm1_clamp_limit = self.quant_config.gemm1_clamp_limit
137
+ if activation == MoEActivation.SILU and gemm1_clamp_limit is not None:
138
+ swiglu_limit_func(output, input, float(gemm1_clamp_limit))
139
+ return
140
+
141
+ super().activation(activation, output, input)
142
+
143
def workspace_shapes(
144
self,
145
M: int,