@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 580353da93
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix] Fix precisions in Gemma 1 (#5913)

Woosuk Kwon committed Jun 28, 2024 at 20:10 UTC 580353da93ee0d96a19964241e16f92e6a6d6142
2 files changed +12 -14
tests/models/test_models.py
+1
@@ -17,6 +17,7 @@ MODELS = [
17 "stabilityai/stablelm-3b-4e1t",
18 # "allenai/OLMo-1B", # Broken
19 "bigcode/starcoder2-3b",
20 + "google/gemma-1.1-2b-it",
21 ]
22
23
vllm/model_executor/models/gemma.py
+11 -14
@@ -26,14 +26,14 @@ from vllm.config import CacheConfig, LoRAConfig
26 from vllm.distributed import get_tensor_model_parallel_world_size
27 from vllm.logger import init_logger
28 from vllm.model_executor.layers.activation import GeluAndMul
29 -from vllm.model_executor.layers.layernorm import RMSNorm
29 +from vllm.model_executor.layers.layernorm import GemmaRMSNorm
30 from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
31 QKVParallelLinear,
32 RowParallelLinear)
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.quantization.base_config import (
35 QuantizationConfig)
36 -from vllm.model_executor.layers.rotary_embedding import get_rope
36 +from vllm.model_executor.layers.rotary_embedding import GemmaRotaryEmbedding
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding)
@@ -148,12 +148,14 @@ class GemmaAttention(nn.Module):
148 quant_config=quant_config,
149 )
150
151 - self.rotary_emb = get_rope(
151 + # TODO(woosuk): Use the `get_rope` interface.
152 + self.rotary_emb = GemmaRotaryEmbedding(
153 self.head_dim,
154 rotary_dim=self.head_dim,
154 - max_position=max_position_embeddings,
155 + max_position_embeddings=max_position_embeddings,
156 base=self.rope_theta,
157 is_neox_style=True,
158 + dtype=torch.get_default_dtype(),
159 )
160 self.attn = Attention(self.num_heads,
161 self.head_dim,
@@ -204,10 +206,10 @@ class GemmaDecoderLayer(nn.Module):
206 hidden_activation=getattr(config, "hidden_activation", None),
207 quant_config=quant_config,
208 )
207 - self.input_layernorm = RMSNorm(config.hidden_size,
208 - eps=config.rms_norm_eps)
209 - self.post_attention_layernorm = RMSNorm(config.hidden_size,
210 - eps=config.rms_norm_eps)
209 + self.input_layernorm = GemmaRMSNorm(config.hidden_size,
210 + eps=config.rms_norm_eps)
211 + self.post_attention_layernorm = GemmaRMSNorm(config.hidden_size,
212 + eps=config.rms_norm_eps)
213
214 def forward(
215 self,
@@ -257,7 +259,7 @@ class GemmaModel(nn.Module):
259 GemmaDecoderLayer(config, cache_config, quant_config)
260 for _ in range(config.num_hidden_layers)
261 ])
260 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
262 + self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
263
264 # Normalize the embedding by sqrt(hidden_size)
265 # The normalizer's data type should be downcasted to the model's
@@ -331,7 +333,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA):
333 self.logits_processor = LogitsProcessor(config.vocab_size)
334 self.sampler = Sampler()
335
334 - @torch.no_grad()
336 def forward(
337 self,
338 input_ids: torch.Tensor,
@@ -388,10 +389,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA):
389 # Skip loading extra bias for GPTQ models.
390 if name.endswith(".bias") and name not in params_dict:
391 continue
391 - # GemmaRMSNorm is different from Llama's in that it multiplies
392 - # (1 + weight) to the output, instead of just weight.
393 - if "norm.weight" in name:
394 - loaded_weight += 1.0
392 param = params_dict[name]
393 weight_loader = getattr(param, "weight_loader",
394 default_weight_loader)