A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Bugfix] More faithful implementation of Gemma (#3653)
Woosuk Kwon committed
Mar 27, 2024 at 09:37 UTC
82c540bebf599e3208683bebf87db56175145ebd
1 file changed
+43
-3
vllm/model_executor/models/gemma.py
+43
-3
@@ -14,6 +14,7 @@
14
# See the License for the specific language governing permissions and
15
# limitations under the License.
16
"""Inference-only Gemma model compatible with HuggingFace weights."""
17
+from functools import lru_cache
18
from typing import List, Optional, Tuple
19
20
import torch
@@ -22,6 +23,7 @@ from transformers import GemmaConfig
23
24
from vllm.attention import Attention, AttentionMetadata
25
from vllm.config import LoRAConfig
26
+from vllm.logger import init_logger
27
from vllm.model_executor.layers.activation import GeluAndMul
28
from vllm.model_executor.layers.layernorm import RMSNorm
29
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -40,6 +42,34 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
42
hf_model_weights_iterator)
43
from vllm.sequence import SamplerOutput
44
45
+logger = init_logger(__name__)
46
+
47
+
48
+@lru_cache(maxsize=None)
49
+def _get_gemma_act_fn(
50
+ hidden_act: Optional[str],
51
+ hidden_activation: Optional[str],
52
+) -> nn.Module:
53
+ if hidden_activation is None:
54
+ if hidden_act is not None:
55
+ logger.warning(
56
+ "Gemma's activation function was incorrectly set to exact GeLU "
57
+ "in the config JSON file when it was initially released. "
58
+ "Changing the activation function to approximate GeLU "
59
+ "(`gelu_pytorch_tanh`). If you want to use the legacy "
60
+ f"`{hidden_act}`, edit the config JSON to set "
61
+ f"`hidden_activation={hidden_act}` instead of `hidden_act`. "
62
+ "See https://github.com/huggingface/transformers/pull/29402 "
63
+ "for more details.")
64
+ return GeluAndMul(approximate="tanh")
65
+ elif hidden_activation == "gelu_pytorch_tanh":
66
+ return GeluAndMul(approximate="tanh")
67
+ elif hidden_activation == "gelu":
68
+ return GeluAndMul(approximate="none")
69
+ else:
70
+ raise ValueError(f"Activation function {hidden_act} is not "
71
+ "supported for Gemma models.")
72
+
73
74
class GemmaMLP(nn.Module):
75
@@ -47,6 +77,8 @@ class GemmaMLP(nn.Module):
77
self,
78
hidden_size: int,
79
intermediate_size: int,
80
+ hidden_act: Optional[str] = None,
81
+ hidden_activation: Optional[str] = None,
82
linear_method: Optional[LinearMethodBase] = None,
83
) -> None:
84
super().__init__()
@@ -58,7 +90,7 @@ class GemmaMLP(nn.Module):
90
hidden_size,
91
bias=False,
92
linear_method=linear_method)
61
- self.act_fn = GeluAndMul()
93
+ self.act_fn = _get_gemma_act_fn(hidden_act, hidden_activation)
94
95
def forward(self, x):
96
gate_up, _ = self.gate_up_proj(x)
@@ -162,6 +194,8 @@ class GemmaDecoderLayer(nn.Module):
194
self.mlp = GemmaMLP(
195
hidden_size=self.hidden_size,
196
intermediate_size=config.intermediate_size,
197
+ hidden_act=config.hidden_act,
198
+ hidden_activation=getattr(config, "hidden_activation", None),
199
linear_method=linear_method,
200
)
201
self.input_layernorm = RMSNorm(config.hidden_size,
@@ -218,6 +252,13 @@ class GemmaModel(nn.Module):
252
])
253
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
254
255
+ # Normalize the embedding by sqrt(hidden_size)
256
+ # The normalizer's data type should be downcasted to the model's
257
+ # data type such as bfloat16, not float32.
258
+ # See https://github.com/huggingface/transformers/pull/29402
259
+ normalizer = self.config.hidden_size**0.5
260
+ self.register_buffer("normalizer", torch.tensor(normalizer))
261
+
262
def forward(
263
self,
264
input_ids: torch.Tensor,
@@ -226,8 +267,7 @@ class GemmaModel(nn.Module):
267
attn_metadata: AttentionMetadata,
268
) -> torch.Tensor:
269
hidden_states = self.embed_tokens(input_ids)
229
- # Normalize the embedding by sqrt(hidden_size)
230
- hidden_states *= self.config.hidden_size**0.5
270
+ hidden_states *= self.normalizer
271
272
residual = None
273
for i in range(len(self.layers)):