A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] Fix precisions in Gemma 1 (#5913)
Woosuk Kwon committed
Jun 28, 2024 at 20:10 UTC
580353da93ee0d96a19964241e16f92e6a6d6142
2 files changed
+12
-14
tests/models/test_models.py
+1
@@ -17,6 +17,7 @@ MODELS = [
17
"stabilityai/stablelm-3b-4e1t",
18
# "allenai/OLMo-1B", # Broken
19
"bigcode/starcoder2-3b",
20
+ "google/gemma-1.1-2b-it",
21
]
22
23
vllm/model_executor/models/gemma.py
+11
-14
@@ -26,14 +26,14 @@ from vllm.config import CacheConfig, LoRAConfig
26
from vllm.distributed import get_tensor_model_parallel_world_size
27
from vllm.logger import init_logger
28
from vllm.model_executor.layers.activation import GeluAndMul
29
-from vllm.model_executor.layers.layernorm import RMSNorm
29
+from vllm.model_executor.layers.layernorm import GemmaRMSNorm
30
from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
31
QKVParallelLinear,
32
RowParallelLinear)
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.quantization.base_config import (
35
QuantizationConfig)
36
-from vllm.model_executor.layers.rotary_embedding import get_rope
36
+from vllm.model_executor.layers.rotary_embedding import GemmaRotaryEmbedding
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding)
@@ -148,12 +148,14 @@ class GemmaAttention(nn.Module):
148
quant_config=quant_config,
149
)
150
151
- self.rotary_emb = get_rope(
151
+ # TODO(woosuk): Use the `get_rope` interface.
152
+ self.rotary_emb = GemmaRotaryEmbedding(
153
self.head_dim,
154
rotary_dim=self.head_dim,
154
- max_position=max_position_embeddings,
155
+ max_position_embeddings=max_position_embeddings,
156
base=self.rope_theta,
157
is_neox_style=True,
158
+ dtype=torch.get_default_dtype(),
159
)
160
self.attn = Attention(self.num_heads,
161
self.head_dim,
@@ -204,10 +206,10 @@ class GemmaDecoderLayer(nn.Module):
206
hidden_activation=getattr(config, "hidden_activation", None),
207
quant_config=quant_config,
208
)
207
- self.input_layernorm = RMSNorm(config.hidden_size,
208
- eps=config.rms_norm_eps)
209
- self.post_attention_layernorm = RMSNorm(config.hidden_size,
210
- eps=config.rms_norm_eps)
209
+ self.input_layernorm = GemmaRMSNorm(config.hidden_size,
210
+ eps=config.rms_norm_eps)
211
+ self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size,
212
+ eps=config.rms_norm_eps)
213
214
def forward(
215
self,
@@ -257,7 +259,7 @@ class GemmaModel(nn.Module):
259
GemmaDecoderLayer(config, cache_config, quant_config)
260
for _ in range(config.num_hidden_layers)
261
])
260
- self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
262
+ self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
263
264
# Normalize the embedding by sqrt(hidden_size)
265
# The normalizer's data type should be downcasted to the model's
@@ -331,7 +333,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA):
333
self.logits_processor = LogitsProcessor(config.vocab_size)
334
self.sampler = Sampler()
335
334
- @torch.no_grad()
336
def forward(
337
self,
338
input_ids: torch.Tensor,
@@ -388,10 +389,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA):
389
# Skip loading extra bias for GPTQ models.
390
if name.endswith(".bias") and name not in params_dict:
391
continue
391
- # GemmaRMSNorm is different from Llama's in that it multiplies
392
- # (1 + weight) to the output, instead of just weight.
393
- if "norm.weight" in name:
394
- loaded_weight += 1.0
392
param = params_dict[name]
393
weight_loader = getattr(param, "weight_loader",
394
default_weight_loader)