A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Use Llama RMSNorm custom op for Gemma (#2974)
Woosuk Kwon committed
Feb 21, 2024 at 18:28 UTC
95529e32537287831cddd800280a20d7c2417163
1 file changed
+27
-33
vllm/model_executor/models/gemma.py
+27
-33
@@ -22,6 +22,7 @@ from transformers import GemmaConfig
22
23
from vllm.model_executor.input_metadata import InputMetadata
24
from vllm.model_executor.layers.attention import PagedAttention
25
+from vllm.model_executor.layers.layernorm import RMSNorm
26
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
27
LinearMethodBase,
28
QKVParallelLinear,
@@ -40,21 +41,6 @@ from vllm.sequence import SamplerOutput
41
KVCache = Tuple[torch.Tensor, torch.Tensor]
42
43
43
-class GemmaRMSNorm(nn.Module):
44
-
45
- def __init__(self, dim: int, eps: float = 1e-6):
46
- super().__init__()
47
- self.eps = eps
48
- self.weight = nn.Parameter(torch.zeros(dim))
49
-
50
- def _norm(self, x):
51
- return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
52
-
53
- def forward(self, x):
54
- output = self._norm(x.float()).type_as(x)
55
- return output * (1 + self.weight)
56
-
57
-
44
class GemmaMLP(nn.Module):
45
46
def __init__(
@@ -185,10 +171,10 @@ class GemmaDecoderLayer(nn.Module):
171
intermediate_size=config.intermediate_size,
172
linear_method=linear_method,
173
)
188
- self.input_layernorm = GemmaRMSNorm(config.hidden_size,
189
- eps=config.rms_norm_eps)
190
- self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size,
191
- eps=config.rms_norm_eps)
174
+ self.input_layernorm = RMSNorm(config.hidden_size,
175
+ eps=config.rms_norm_eps)
176
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
177
+ eps=config.rms_norm_eps)
178
179
def forward(
180
self,
@@ -196,25 +182,27 @@ class GemmaDecoderLayer(nn.Module):
182
hidden_states: torch.Tensor,
183
kv_cache: KVCache,
184
input_metadata: InputMetadata,
185
+ residual: Optional[torch.Tensor],
186
) -> Tuple[torch.Tensor, torch.Tensor]:
187
# Self Attention
201
- residual = hidden_states
202
- hidden_states = self.input_layernorm(hidden_states)
188
+ if residual is None:
189
+ residual = hidden_states
190
+ hidden_states = self.input_layernorm(hidden_states)
191
+ else:
192
+ hidden_states, residual = self.input_layernorm(
193
+ hidden_states, residual)
194
hidden_states = self.self_attn(
195
positions=positions,
196
hidden_states=hidden_states,
197
kv_cache=kv_cache,
198
input_metadata=input_metadata,
199
)
209
- hidden_states = residual + hidden_states
200
201
# Fully Connected
212
- residual = hidden_states
213
- hidden_states = self.post_attention_layernorm(hidden_states)
202
+ hidden_states, residual = self.post_attention_layernorm(
203
+ hidden_states, residual)
204
hidden_states = self.mlp(hidden_states)
215
- hidden_states = residual + hidden_states
216
-
217
- return hidden_states
205
+ return hidden_states, residual
206
207
208
class GemmaModel(nn.Module):
@@ -235,7 +223,7 @@ class GemmaModel(nn.Module):
223
GemmaDecoderLayer(config, linear_method)
224
for _ in range(config.num_hidden_layers)
225
])
238
- self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
226
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
227
228
def forward(
229
self,
@@ -246,17 +234,19 @@ class GemmaModel(nn.Module):
234
) -> torch.Tensor:
235
hidden_states = self.embed_tokens(input_ids)
236
# Normalize the embedding by sqrt(hidden_size)
249
- hidden_states = hidden_states * (self.config.hidden_size**0.5)
237
+ hidden_states *= self.config.hidden_size**0.5
238
239
+ residual = None
240
for i in range(len(self.layers)):
241
layer = self.layers[i]
253
- hidden_states = layer(
242
+ hidden_states, residual = layer(
243
positions,
244
hidden_states,
245
kv_caches[i],
246
input_metadata,
247
+ residual,
248
)
259
- hidden_states = self.norm(hidden_states)
249
+ hidden_states, _ = self.norm(hidden_states, residual)
250
return hidden_states
251
252
@@ -321,6 +311,10 @@ class GemmaForCausalLM(nn.Module):
311
# Skip loading extra layer for lora models.
312
if "lm_head" in name:
313
continue
314
+ # GemmaRMSNorm is different from Llama's in that it multiplies
315
+ # (1 + weight) to the output, instead of just weight.
316
+ if "norm.weight" in name:
317
+ loaded_weight += 1.0
318
param = params_dict[name]
319
weight_loader = getattr(param, "weight_loader",
320
default_weight_loader)
@@ -329,5 +323,5 @@ class GemmaForCausalLM(nn.Module):
323
unloaded_params = params_dict.keys() - loaded_params
324
if unloaded_params:
325
raise RuntimeError(
332
- f"Some weights are not initialized from checkpoints: {unloaded_params}"
333
- )
326
+ "Some weights are not initialized from checkpoints: "
327
+ f"{unloaded_params}")