@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5b8a1fde84
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model][Bugfix] Add FATReLU activation and support for openbmb/MiniCPM-S-1B-sft (#9396)

Junhao Li committed Oct 16, 2024 at 12:40 UTC 5b8a1fde84224e24ec121e0dc149d775330d911b
3 files changed +37 -5
docs/source/models/supported_models.rst
+1 -1
@@ -159,7 +159,7 @@ Text Generation
159 -
160 * - :code:`MiniCPMForCausalLM`
161 - MiniCPM
162 - - :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, etc.
162 + - :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, :code:`openbmb/MiniCPM-S-1B-sft`, etc.
163 - ✅︎
164 - ✅︎
165 * - :code:`MiniCPM3ForCausalLM`
vllm/model_executor/layers/activation.py
+27
@@ -13,6 +13,33 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
13 from vllm.model_executor.utils import set_weight_attrs
14
15
16 +class FatreluAndMul(CustomOp):
17 + """An activation function for FATReLU.
18 +
19 + The function computes x -> FATReLU(x[:d]) * x[d:] where
20 + d = x.shape[-1] // 2.
21 + This is used in openbmb/MiniCPM-S-1B-sft.
22 +
23 + Shapes:
24 + x: (num_tokens, 2 * d) or (batch_size, seq_len, 2 * d)
25 + return: (num_tokens, d) or (batch_size, seq_len, d)
26 + """
27 +
28 + def __init__(self, threshold: float = 0.):
29 + super().__init__()
30 + self.threshold = threshold
31 +
32 + def forward_native(self, x: torch.Tensor) -> torch.Tensor:
33 + d = x.shape[-1] // 2
34 + x1 = x[..., :d]
35 + x2 = x[..., d:]
36 + x1 = F.threshold(x1, self.threshold, 0.0)
37 + return x1 * x2
38 +
39 + def forward_cuda(self, x: torch.Tensor) -> torch.Tensor:
40 + return self.forward_native(x)
41 +
42 +
43 class SiluAndMul(CustomOp):
44 """An activation function for SwiGLU.
45
vllm/model_executor/models/minicpm.py
+9 -4
@@ -33,7 +33,7 @@ from vllm.config import CacheConfig, LoRAConfig
33 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size,
35 tensor_model_parallel_all_reduce)
36 -from vllm.model_executor.layers.activation import SiluAndMul
36 +from vllm.model_executor.layers.activation import FatreluAndMul, SiluAndMul
37 from vllm.model_executor.layers.fused_moe import fused_moe
38 from vllm.model_executor.layers.layernorm import RMSNorm
39 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
@@ -152,6 +152,7 @@ class MiniCPMMLP(nn.Module):
152 hidden_size: int,
153 intermediate_size: int,
154 hidden_act: str,
155 + hidden_act_param: float,
156 quant_config: Optional[QuantizationConfig] = None,
157 ) -> None:
158 super().__init__()
@@ -163,10 +164,13 @@ class MiniCPMMLP(nn.Module):
164 hidden_size,
165 bias=False,
166 quant_config=quant_config)
166 - if hidden_act != "silu":
167 + if hidden_act == "silu":
168 + self.act_fn = SiluAndMul()
169 + elif hidden_act == "fatrelu":
170 + self.act_fn = FatreluAndMul(threshold=hidden_act_param)
171 + else:
172 raise ValueError(f"Unsupported activation: {hidden_act}. "
168 - "Only silu is supported for now.")
169 - self.act_fn = SiluAndMul()
173 + "Only silu and fatrelu are supported for now.")
174
175 def forward(self, x):
176 gate_up, _ = self.gate_up_proj(x)
@@ -304,6 +308,7 @@ class MiniCPMDecoderLayer(nn.Module):
308 hidden_size=self.hidden_size,
309 intermediate_size=self.config.intermediate_size,
310 hidden_act=self.config.hidden_act,
311 + hidden_act_param=getattr(self.config, "hidden_act_param", 0.),
312 quant_config=self.quant_config,
313 )
314 else: