A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 01:30 UTC
[Chore] Replace swish with silu (#32459)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 16, 2026 at 16:22 UTC
180e981d567a3f6db76078b210b582e59d10672b
1 file changed
+9
-45
vllm/model_executor/models/phi4mm_utils.py
+9
-45
@@ -36,10 +36,13 @@ def get_activation(name: str = "relu") -> torch.nn.Module:
36
if name == "gelu":
37
return nn.GELU()
38
if name == "swish":
39
- return Swish()
39
+ return nn.SiLU()
40
if name == "sigmoid":
41
- return torch.nn.Sigmoid()
42
- return nn.Identity()
41
+ return nn.Sigmoid()
42
+ if name == "identity":
43
+ return nn.Identity()
44
+
45
+ raise NotImplementedError(name)
46
47
48
def adaptive_enc_mask(
@@ -93,44 +96,14 @@ def adaptive_enc_mask(
96
return mask_left & mask_right
97
98
96
-class Swish(nn.Module):
97
- """Implement Swish activation module.
98
- From https://arxiv.org/pdf/2005.03191.pdf
99
-
100
- """
101
-
102
- def __init__(self) -> None:
103
- super().__init__()
104
- self.act_fn = nn.Sigmoid()
105
-
106
- def forward(self, x: Tensor) -> Tensor:
107
- """Apply Swish function
108
-
109
- Args:
110
- x: torch.Tensor
111
- Input.
112
- """
113
- return x * self.act_fn(x)
114
-
115
-
99
class GLU(nn.Module):
100
"""Implement Gated Linear Unit (GLU) module"""
101
102
def __init__(self, dim: int = -1, act_name: str = "sigmoid") -> None:
103
super().__init__()
104
+
105
self.dim = dim
122
- self.act_name = act_name.lower()
123
-
124
- if self.act_name == "relu":
125
- self.act_fn = nn.ReLU(inplace=True)
126
- elif self.act_name == "gelu":
127
- self.act_fn = nn.GELU()
128
- elif self.act_name == "swish":
129
- self.act_fn = Swish()
130
- elif self.act_name == "sigmoid":
131
- self.act_fn = nn.Sigmoid()
132
- else:
133
- self.act_fn = nn.Identity()
106
+ self.act_fn = get_activation(act_name)
107
108
def forward(self, x: Tensor) -> Tensor:
109
"""GLU forward
@@ -204,16 +177,7 @@ class GLUPointWiseConv(nn.Module):
177
padding=(kernel_size - 1) // 2,
178
)
179
207
- if glu_type == "sigmoid":
208
- self.glu_act = nn.Sigmoid()
209
- elif glu_type == "relu":
210
- self.glu_act = nn.ReLU()
211
- elif glu_type == "gelu":
212
- self.glu_act = nn.GELU()
213
- elif glu_type == "swish":
214
- self.glu_act = Swish()
215
- else:
216
- raise ValueError(f"Unsupported activation type {self.glu_act}")
180
+ self.glu_act = get_activation(glu_type)
181
182
if bias_in_glu:
183
self.b1 = nn.Parameter(torch.zeros(1, output_dim, 1))