@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 82c540bebf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Bugfix] More faithful implementation of Gemma (#3653)

Woosuk Kwon committed Mar 27, 2024 at 09:37 UTC 82c540bebf599e3208683bebf87db56175145ebd
1 file changed +43 -3
vllm/model_executor/models/gemma.py
+43 -3
@@ -14,6 +14,7 @@
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 """Inference-only Gemma model compatible with HuggingFace weights."""
17 +from functools import lru_cache
18 from typing import List, Optional, Tuple
19
20 import torch
@@ -22,6 +23,7 @@ from transformers import GemmaConfig
23
24 from vllm.attention import Attention, AttentionMetadata
25 from vllm.config import LoRAConfig
26 +from vllm.logger import init_logger
27 from vllm.model_executor.layers.activation import GeluAndMul
28 from vllm.model_executor.layers.layernorm import RMSNorm
29 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -40,6 +42,34 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
42 hf_model_weights_iterator)
43 from vllm.sequence import SamplerOutput
44
45 +logger = init_logger(__name__)
46 +
47 +
48 +@lru_cache(maxsize=None)
49 +def _get_gemma_act_fn(
50 + hidden_act: Optional[str],
51 + hidden_activation: Optional[str],
52 +) -> nn.Module:
53 + if hidden_activation is None:
54 + if hidden_act is not None:
55 + logger.warning(
56 + "Gemma's activation function was incorrectly set to exact GeLU "
57 + "in the config JSON file when it was initially released. "
58 + "Changing the activation function to approximate GeLU "
59 + "(`gelu_pytorch_tanh`). If you want to use the legacy "
60 + f"`{hidden_act}`, edit the config JSON to set "
61 + f"`hidden_activation={hidden_act}` instead of `hidden_act`. "
62 + "See https://github.com/huggingface/transformers/pull/29402 "
63 + "for more details.")
64 + return GeluAndMul(approximate="tanh")
65 + elif hidden_activation == "gelu_pytorch_tanh":
66 + return GeluAndMul(approximate="tanh")
67 + elif hidden_activation == "gelu":
68 + return GeluAndMul(approximate="none")
69 + else:
70 + raise ValueError(f"Activation function {hidden_act} is not "
71 + "supported for Gemma models.")
72 +
73
74 class GemmaMLP(nn.Module):
75
@@ -47,6 +77,8 @@ class GemmaMLP(nn.Module):
77 self,
78 hidden_size: int,
79 intermediate_size: int,
80 + hidden_act: Optional[str] = None,
81 + hidden_activation: Optional[str] = None,
82 linear_method: Optional[LinearMethodBase] = None,
83 ) -> None:
84 super().__init__()
@@ -58,7 +90,7 @@ class GemmaMLP(nn.Module):
90 hidden_size,
91 bias=False,
92 linear_method=linear_method)
61 - self.act_fn = GeluAndMul()
93 + self.act_fn = _get_gemma_act_fn(hidden_act, hidden_activation)
94
95 def forward(self, x):
96 gate_up, _ = self.gate_up_proj(x)
@@ -162,6 +194,8 @@ class GemmaDecoderLayer(nn.Module):
194 self.mlp = GemmaMLP(
195 hidden_size=self.hidden_size,
196 intermediate_size=config.intermediate_size,
197 + hidden_act=config.hidden_act,
198 + hidden_activation=getattr(config, "hidden_activation", None),
199 linear_method=linear_method,
200 )
201 self.input_layernorm = RMSNorm(config.hidden_size,
@@ -218,6 +252,13 @@ class GemmaModel(nn.Module):
252 ])
253 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
254
255 + # Normalize the embedding by sqrt(hidden_size)
256 + # The normalizer's data type should be downcasted to the model's
257 + # data type such as bfloat16, not float32.
258 + # See https://github.com/huggingface/transformers/pull/29402
259 + normalizer = self.config.hidden_size**0.5
260 + self.register_buffer("normalizer", torch.tensor(normalizer))
261 +
262 def forward(
263 self,
264 input_ids: torch.Tensor,
@@ -226,8 +267,7 @@ class GemmaModel(nn.Module):
267 attn_metadata: AttentionMetadata,
268 ) -> torch.Tensor:
269 hidden_states = self.embed_tokens(input_ids)
229 - # Normalize the embedding by sqrt(hidden_size)
230 - hidden_states *= self.config.hidden_size**0.5
270 + hidden_states *= self.normalizer
271
272 residual = None
273 for i in range(len(self.layers)):