@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / df845b2b46
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Misc] Remove Gemma RoPE (#7638)

Woosuk Kwon committed Aug 19, 2024 at 09:29 UTC df845b2b46c3e30f5bd3e3be286285ed148323fc
3 files changed +7 -26
vllm/model_executor/layers/rotary_embedding.py
-15
@@ -93,11 +93,6 @@ class RotaryEmbedding(CustomOp):
93
94 def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
95 """Compute the inverse frequency."""
96 - # NOTE(woosuk): The HF implementation uses `torch.arange(...).float()`.
97 - # However, we use `torch.arange(..., dtype=torch.float)` instead to
98 - # avoid numerical issues with large base values (e.g., 10000000).
99 - # This may cause a slight numerical difference between the HF
100 - # implementation and ours.
96 # NOTE(woosuk): To exactly match the HF implementation, we need to
97 # use CPU to compute the cache and then move it to GPU. However, we
98 # create the cache on GPU for faster initialization. This may cause
@@ -724,16 +719,6 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbedding):
719 return query, key
720
721
727 -class GemmaRotaryEmbedding(RotaryEmbedding):
728 -
729 - def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
730 - # https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/gemma/modeling_gemma.py#L107
731 - inv_freq = 1.0 / (base**(
732 - torch.arange(0, self.rotary_dim, 2, dtype=torch.int64).float() /
733 - self.rotary_dim))
734 - return inv_freq
735 -
736 -
722 class Llama3RotaryEmbedding(RotaryEmbedding):
723
724 def __init__(
vllm/model_executor/models/gemma.py
+3 -5
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
33 from vllm.model_executor.layers.logits_processor import LogitsProcessor
34 from vllm.model_executor.layers.quantization.base_config import (
35 QuantizationConfig)
36 -from vllm.model_executor.layers.rotary_embedding import GemmaRotaryEmbedding
36 +from vllm.model_executor.layers.rotary_embedding import get_rope
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding)
@@ -148,14 +148,12 @@ class GemmaAttention(nn.Module):
148 quant_config=quant_config,
149 )
150
151 - # TODO(woosuk): Use the `get_rope` interface.
152 - self.rotary_emb = GemmaRotaryEmbedding(
151 + self.rotary_emb = get_rope(
152 self.head_dim,
153 rotary_dim=self.head_dim,
155 - max_position_embeddings=max_position_embeddings,
154 + max_position=max_position_embeddings,
155 base=self.rope_theta,
156 is_neox_style=True,
158 - dtype=torch.get_default_dtype(),
157 )
158 self.attn = Attention(self.num_heads,
159 self.head_dim,
vllm/model_executor/models/gemma2.py
+4 -6
@@ -32,7 +32,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
32 from vllm.model_executor.layers.logits_processor import LogitsProcessor
33 from vllm.model_executor.layers.quantization.base_config import (
34 QuantizationConfig)
35 -from vllm.model_executor.layers.rotary_embedding import GemmaRotaryEmbedding
35 +from vllm.model_executor.layers.rotary_embedding import get_rope
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding)
@@ -130,14 +130,12 @@ class Gemma2Attention(nn.Module):
130 bias=config.attention_bias,
131 quant_config=quant_config,
132 )
133 - # TODO(woosuk): Use the `get_rope` interface.
134 - self.rotary_emb = GemmaRotaryEmbedding(
133 + self.rotary_emb = get_rope(
134 self.head_dim,
136 - self.head_dim,
137 - max_position_embeddings,
135 + rotary_dim=self.head_dim,
136 + max_position=max_position_embeddings,
137 base=self.rope_theta,
138 is_neox_style=True,
140 - dtype=torch.get_default_dtype(),
139 )
140
141 # FIXME(woosuk): While Gemma 2 uses sliding window attention for every