@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 95529e3253
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Use Llama RMSNorm custom op for Gemma (#2974)

Woosuk Kwon committed Feb 21, 2024 at 18:28 UTC 95529e32537287831cddd800280a20d7c2417163
1 file changed +27 -33
vllm/model_executor/models/gemma.py
+27 -33
@@ -22,6 +22,7 @@ from transformers import GemmaConfig
22
23 from vllm.model_executor.input_metadata import InputMetadata
24 from vllm.model_executor.layers.attention import PagedAttention
25 +from vllm.model_executor.layers.layernorm import RMSNorm
26 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
27 LinearMethodBase,
28 QKVParallelLinear,
@@ -40,21 +41,6 @@ from vllm.sequence import SamplerOutput
41 KVCache = Tuple[torch.Tensor, torch.Tensor]
42
43
43 -class GemmaRMSNorm(nn.Module):
44 -
45 - def __init__(self, dim: int, eps: float = 1e-6):
46 - super().__init__()
47 - self.eps = eps
48 - self.weight = nn.Parameter(torch.zeros(dim))
49 -
50 - def _norm(self, x):
51 - return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
52 -
53 - def forward(self, x):
54 - output = self._norm(x.float()).type_as(x)
55 - return output * (1 + self.weight)
56 -
57 -
44 class GemmaMLP(nn.Module):
45
46 def __init__(
@@ -185,10 +171,10 @@ class GemmaDecoderLayer(nn.Module):
171 intermediate_size=config.intermediate_size,
172 linear_method=linear_method,
173 )
188 - self.input_layernorm = GemmaRMSNorm(config.hidden_size,
189 - eps=config.rms_norm_eps)
190 - self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size,
191 - eps=config.rms_norm_eps)
174 + self.input_layernorm = RMSNorm(config.hidden_size,
175 + eps=config.rms_norm_eps)
176 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
177 + eps=config.rms_norm_eps)
178
179 def forward(
180 self,
@@ -196,25 +182,27 @@ class GemmaDecoderLayer(nn.Module):
182 hidden_states: torch.Tensor,
183 kv_cache: KVCache,
184 input_metadata: InputMetadata,
185 + residual: Optional[torch.Tensor],
186 ) -> Tuple[torch.Tensor, torch.Tensor]:
187 # Self Attention
201 - residual = hidden_states
202 - hidden_states = self.input_layernorm(hidden_states)
188 + if residual is None:
189 + residual = hidden_states
190 + hidden_states = self.input_layernorm(hidden_states)
191 + else:
192 + hidden_states, residual = self.input_layernorm(
193 + hidden_states, residual)
194 hidden_states = self.self_attn(
195 positions=positions,
196 hidden_states=hidden_states,
197 kv_cache=kv_cache,
198 input_metadata=input_metadata,
199 )
209 - hidden_states = residual + hidden_states
200
201 # Fully Connected
212 - residual = hidden_states
213 - hidden_states = self.post_attention_layernorm(hidden_states)
202 + hidden_states, residual = self.post_attention_layernorm(
203 + hidden_states, residual)
204 hidden_states = self.mlp(hidden_states)
215 - hidden_states = residual + hidden_states
216 -
217 - return hidden_states
205 + return hidden_states, residual
206
207
208 class GemmaModel(nn.Module):
@@ -235,7 +223,7 @@ class GemmaModel(nn.Module):
223 GemmaDecoderLayer(config, linear_method)
224 for _ in range(config.num_hidden_layers)
225 ])
238 - self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
226 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
227
228 def forward(
229 self,
@@ -246,17 +234,19 @@ class GemmaModel(nn.Module):
234 ) -> torch.Tensor:
235 hidden_states = self.embed_tokens(input_ids)
236 # Normalize the embedding by sqrt(hidden_size)
249 - hidden_states = hidden_states * (self.config.hidden_size**0.5)
237 + hidden_states *= self.config.hidden_size**0.5
238
239 + residual = None
240 for i in range(len(self.layers)):
241 layer = self.layers[i]
253 - hidden_states = layer(
242 + hidden_states, residual = layer(
243 positions,
244 hidden_states,
245 kv_caches[i],
246 input_metadata,
247 + residual,
248 )
259 - hidden_states = self.norm(hidden_states)
249 + hidden_states, _ = self.norm(hidden_states, residual)
250 return hidden_states
251
252
@@ -321,6 +311,10 @@ class GemmaForCausalLM(nn.Module):
311 # Skip loading extra layer for lora models.
312 if "lm_head" in name:
313 continue
314 + # GemmaRMSNorm is different from Llama's in that it multiplies
315 + # (1 + weight) to the output, instead of just weight.
316 + if "norm.weight" in name:
317 + loaded_weight += 1.0
318 param = params_dict[name]
319 weight_loader = getattr(param, "weight_loader",
320 default_weight_loader)
@@ -329,5 +323,5 @@ class GemmaForCausalLM(nn.Module):
323 unloaded_params = params_dict.keys() - loaded_params
324 if unloaded_params:
325 raise RuntimeError(
332 - f"Some weights are not initialized from checkpoints: {unloaded_params}"
333 - )
326 + "Some weights are not initialized from checkpoints: "
327 + f"{unloaded_params}")