A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model][Bugfix] Add FATReLU activation and support for openbmb/MiniCPM-S-1B-sft (#9396)
Junhao Li committed
Oct 16, 2024 at 12:40 UTC
5b8a1fde84224e24ec121e0dc149d775330d911b
3 files changed
+37
-5
docs/source/models/supported_models.rst
+1
-1
@@ -159,7 +159,7 @@ Text Generation
159
-
160
* - :code:`MiniCPMForCausalLM`
161
- MiniCPM
162
- - :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, etc.
162
+ - :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, :code:`openbmb/MiniCPM-S-1B-sft`, etc.
163
- ✅︎
164
- ✅︎
165
* - :code:`MiniCPM3ForCausalLM`
vllm/model_executor/layers/activation.py
+27
@@ -13,6 +13,33 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
13
from vllm.model_executor.utils import set_weight_attrs
14
15
16
+class FatreluAndMul(CustomOp):
17
+ """An activation function for FATReLU.
18
+
19
+ The function computes x -> FATReLU(x[:d]) * x[d:] where
20
+ d = x.shape[-1] // 2.
21
+ This is used in openbmb/MiniCPM-S-1B-sft.
22
+
23
+ Shapes:
24
+ x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
25
+ return: (num_tokens, d) or (batch_size, seq_len, d)
26
+ """
27
+
28
+ def __init__(self, threshold: float = 0.):
29
+ super().__init__()
30
+ self.threshold = threshold
31
+
32
+ def forward_native(self, x: torch.Tensor) -> torch.Tensor:
33
+ d = x.shape[-1] // 2
34
+ x1 = x[..., :d]
35
+ x2 = x[..., d:]
36
+ x1 = F.threshold(x1, self.threshold, 0.0)
37
+ return x1 * x2
38
+
39
+ def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
40
+ return self.forward_native(x)
41
+
42
+
43
class SiluAndMul(CustomOp):
44
"""An activation function for SwiGLU.
45
vllm/model_executor/models/minicpm.py
+9
-4
@@ -33,7 +33,7 @@ from vllm.config import CacheConfig, LoRAConfig
33
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size,
35
tensor_model_parallel_all_reduce)
36
-from vllm.model_executor.layers.activation import SiluAndMul
36
+from vllm.model_executor.layers.activation import FatreluAndMul, SiluAndMul
37
from vllm.model_executor.layers.fused_moe import fused_moe
38
from vllm.model_executor.layers.layernorm import RMSNorm
39
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -152,6 +152,7 @@ class MiniCPMMLP(nn.Module):
152
hidden_size: int,
153
intermediate_size: int,
154
hidden_act: str,
155
+ hidden_act_param: float,
156
quant_config: Optional[QuantizationConfig] = None,
157
) -> None:
158
super().__init__()
@@ -163,10 +164,13 @@ class MiniCPMMLP(nn.Module):
164
hidden_size,
165
bias=False,
166
quant_config=quant_config)
166
- if hidden_act != "silu":
167
+ if hidden_act == "silu":
168
+ self.act_fn = SiluAndMul()
169
+ elif hidden_act == "fatrelu":
170
+ self.act_fn = FatreluAndMul(threshold=hidden_act_param)
171
+ else:
172
raise ValueError(f"Unsupported activation: {hidden_act}. "
168
- "Only silu is supported for now.")
169
- self.act_fn = SiluAndMul()
173
+ "Only silu and fatrelu are supported for now.")
174
175
def forward(self, x):
176
gate_up, _ = self.gate_up_proj(x)
@@ -304,6 +308,7 @@ class MiniCPMDecoderLayer(nn.Module):
308
hidden_size=self.hidden_size,
309
intermediate_size=self.config.intermediate_size,
310
hidden_act=self.config.hidden_act,
311
+ hidden_act_param=getattr(self.config, "hidden_act_param", 0.),
312
quant_config=self.quant_config,
313
)
314
else: