A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 23:45 UTC
[Qwen][Bugfix] Fixes sigmoid activation in torch impl of RMSNormGated. (#40245)
Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
Tao He committed
Apr 20, 2026 at 12:28 UTC
8936118134d0547fa1cc78adab2d03edd6d3dc48
2 files changed
+13
-2
vllm/model_executor/layers/layernorm.py
+5
-2
@@ -478,9 +478,12 @@ class RMSNormGated(CustomOp):
478
weight = self.weight.float()
479
z = z.float() if z is not None else None
480
481
+ assert self.activation in ["silu", "sigmoid", "swish"]
482
+ act_fn = F.sigmoid if self.activation == "sigmoid" else F.silu
483
+
484
# Apply gating before normalization if needed
485
if z is not None and not self.norm_before_gate:
483
- x = x * F.silu(z)
486
+ x = x * act_fn(z)
487
488
# RMS Normalization
489
if self.group_size is None:
@@ -499,7 +502,7 @@ class RMSNormGated(CustomOp):
502
503
# Apply gating after normalization if needed
504
if z is not None and self.norm_before_gate:
502
- out = out * F.silu(z)
505
+ out = out * act_fn(z)
506
507
return out.to(orig_dtype)
508
vllm/model_executor/layers/mamba/gdn_linear_attn.py
+8
@@ -357,11 +357,19 @@ class GatedDeltaNetAttention(PluggableLayer, MambaBase):
357
set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(0)})
358
set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
359
360
+ output_gate_type = getattr(config, "output_gate_type", "silu")
361
+ if output_gate_type == "swish":
362
+ output_gate_type = "silu"
363
+ assert output_gate_type in ["silu", "swish", "sigmoid"], (
364
+ f"unsupported {output_gate_type=}"
365
+ )
366
+
367
self.norm = RMSNormGated(
368
self.head_v_dim,
369
eps=self.layer_norm_epsilon,
370
group_size=None,
371
norm_before_gate=True,
372
+ activation=output_gate_type,
373
device=current_platform.current_device(),
374
)
375