A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 10:30 UTC
[Bugfix] fix swiglu limit issue for humming backend + deepseek v4 (#42541)
Signed-off-by: Jinzhen Lin <jinzhen.ljz@antgroup.com> Co-authored-by: Michael Goin <mgoin64@gmail.com>
Jinzhen Lin committed
May 19, 2026 at 01:32 UTC
6859ca76159fdd403b687c0c296e5a12850ba24e
3 files changed
+34
-6
vllm/model_executor/layers/fused_moe/experts/fused_humming_moe.py
+19
-4
@@ -33,7 +33,10 @@ from vllm.model_executor.layers.fused_moe.topk_weight_and_reduce import (
33
TopKWeightAndReduceDelegate,
34
TopKWeightAndReduceNoOP,
35
)
36
-from vllm.model_executor.layers.fused_moe.utils import _resize_cache
36
+from vllm.model_executor.layers.fused_moe.utils import (
37
+ _resize_cache,
38
+ swiglu_limit_func,
39
+)
40
from vllm.model_executor.layers.quantization.utils.quant_utils import QuantKey
41
from vllm.platforms import current_platform
42
from vllm.v1.worker.workspace import current_workspace_manager
@@ -422,6 +425,18 @@ class HummingExpertsBase(mk.FusedMoEExpertsModular):
425
426
return supported, None if supported else reason
427
428
+ def apply_activation(
429
+ self,
430
+ activation: MoEActivation,
431
+ output: torch.Tensor,
432
+ input: torch.Tensor,
433
+ ) -> None:
434
+ swiglu_limit = self.quant_config.gemm1_clamp_limit
435
+ if activation == MoEActivation.SILU and swiglu_limit is not None:
436
+ swiglu_limit_func(output=output, input=input, swiglu_limit=swiglu_limit)
437
+ else:
438
+ self.activation(activation=activation, input=input, output=output)
439
+
440
441
class HummingIndexedExperts(HummingExpertsBase):
442
def finalize_weight_and_reduce_impl(self) -> mk.TopKWeightAndReduce:
@@ -514,7 +529,7 @@ class HummingIndexedExperts(HummingExpertsBase):
529
**moe_kwargs1,
530
)
531
517
- self.activation(
532
+ self.apply_activation(
533
activation=self.layer.activation,
534
input=buffers["gate_up_output"],
535
output=buffers["activation_output"],
@@ -604,7 +619,7 @@ class HummingGroupedExperts(HummingExpertsBase):
619
sublayer_name="w13",
620
)
621
607
- self.activation(
622
+ self.apply_activation(
623
activation=self.layer.activation,
624
input=buffers["gate_up_output"],
625
output=buffers["activation_output"],
@@ -691,7 +706,7 @@ class BatchedHummingGroupedExperts(HummingExpertsBase):
706
sublayer_name="w13",
707
)
708
694
- self.activation(
709
+ self.apply_activation(
710
activation=self.layer.activation,
711
input=buffers["gate_up_output"],
712
output=buffers["activation_output"],
vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
+6
-1
@@ -1567,7 +1567,12 @@ def make_mxfp4_moe_quant_config(
1567
)
1568
1569
assert isinstance(layer, FusedMoE)
1570
- return get_humming_moe_quant_config(layer)
1570
+ return get_humming_moe_quant_config(
1571
+ layer,
1572
+ gemm1_alpha=gemm1_alpha,
1573
+ gemm1_beta=gemm1_beta,
1574
+ gemm1_clamp_limit=swiglu_limit,
1575
+ )
1576
else:
1577
return ocp_mx_moe_quant_config(
1578
quant_dtype="mxfp4",
vllm/model_executor/layers/quantization/utils/humming_utils.py
+9
-1
@@ -164,7 +164,12 @@ def prepare_humming_moe_layer(layer: RoutedExperts, quant_config: dict):
164
layer.register_buffer("locks", locks)
165
166
167
-def get_humming_moe_quant_config(layer: RoutedExperts):
167
+def get_humming_moe_quant_config(
168
+ layer: RoutedExperts,
169
+ gemm1_alpha: float | None = None,
170
+ gemm1_beta: float | None = None,
171
+ gemm1_clamp_limit: float | None = None,
172
+):
173
input_schema = layer.input_schemas["w13"]
174
weight_schema = layer.weight_schemas["w13"]
175
@@ -211,4 +216,7 @@ def get_humming_moe_quant_config(layer: RoutedExperts):
216
_a2=a_quant_desc,
217
_w1=w1_quant_desc,
218
_w2=w2_quant_desc,
219
+ gemm1_alpha=gemm1_alpha,
220
+ gemm1_beta=gemm1_beta,
221
+ gemm1_clamp_limit=gemm1_clamp_limit,
222
)