A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 21:30 UTC
[Bugfix] Fix DSV4 swiglu_limit on marlin backend (#42287)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
Jee Jee Li committed
May 12, 2026 at 04:03 UTC
53181384e036cc660ff6754c8a94a8e6d0b60562
1 file changed
+4
vllm/model_executor/layers/fused_moe/experts/marlin_moe.py
+4
@@ -413,6 +413,7 @@ def batched_fused_marlin_moe(
413
is_k_full: bool = True,
414
output: torch.Tensor | None = None,
415
inplace: bool = False,
416
+ clamp_limit: float | None = None,
417
) -> torch.Tensor:
418
"""
419
This function massages the inputs so the batched hidden_states can be
@@ -535,6 +536,7 @@ def batched_fused_marlin_moe(
536
intermediate_cache2=intermediate_cache2,
537
output=output.view(-1, K) if output is not None else output,
538
is_k_full=is_k_full,
539
+ clamp_limit=clamp_limit,
540
)
541
542
output = output.view(B, BATCH_TOKENS_MAX, K)
@@ -768,6 +770,7 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
770
sort_indices2=self.w2_g_idx_sort_indices,
771
is_k_full=self.is_k_full,
772
input_dtype=self.input_dtype,
773
+ clamp_limit=self.gemm1_clamp_limit,
774
)
775
return
776
@@ -970,4 +973,5 @@ class BatchedMarlinExperts(MarlinExpertsBase):
973
sort_indices1=self.w13_g_idx_sort_indices,
974
sort_indices2=self.w2_g_idx_sort_indices,
975
is_k_full=self.is_k_full,
976
+ clamp_limit=self.gemm1_clamp_limit,
977
)