A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Misc] Remove Gemma RoPE (#7638)
Woosuk Kwon committed
Aug 19, 2024 at 09:29 UTC
df845b2b46c3e30f5bd3e3be286285ed148323fc
3 files changed
+7
-26
vllm/model_executor/layers/rotary_embedding.py
-15
@@ -93,11 +93,6 @@ class RotaryEmbedding(CustomOp):
93
94
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
95
"""Compute the inverse frequency."""
96
- # NOTE(woosuk): The HF implementation uses `torch.arange(...).float()`.
97
- # However, we use `torch.arange(..., dtype=torch.float)` instead to
98
- # avoid numerical issues with large base values (e.g., 10000000).
99
- # This may cause a slight numerical difference between the HF
100
- # implementation and ours.
96
# NOTE(woosuk): To exactly match the HF implementation, we need to
97
# use CPU to compute the cache and then move it to GPU. However, we
98
# create the cache on GPU for faster initialization. This may cause
@@ -724,16 +719,6 @@ class DeepseekScalingRotaryEmbedding(RotaryEmbedding):
719
return query, key
720
721
727
-class GemmaRotaryEmbedding(RotaryEmbedding):
728
-
729
- def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
730
- # https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/gemma/modeling_gemma.py#L107
731
- inv_freq = 1.0 / (base**(
732
- torch.arange(0, self.rotary_dim, 2, dtype=torch.int64).float() /
733
- self.rotary_dim))
734
- return inv_freq
735
-
736
-
722
class Llama3RotaryEmbedding(RotaryEmbedding):
723
724
def __init__(
vllm/model_executor/models/gemma.py
+3
-5
@@ -33,7 +33,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
33
from vllm.model_executor.layers.logits_processor import LogitsProcessor
34
from vllm.model_executor.layers.quantization.base_config import (
35
QuantizationConfig)
36
-from vllm.model_executor.layers.rotary_embedding import GemmaRotaryEmbedding
36
+from vllm.model_executor.layers.rotary_embedding import get_rope
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding)
@@ -148,14 +148,12 @@ class GemmaAttention(nn.Module):
148
quant_config=quant_config,
149
)
150
151
- # TODO(woosuk): Use the `get_rope` interface.
152
- self.rotary_emb = GemmaRotaryEmbedding(
151
+ self.rotary_emb = get_rope(
152
self.head_dim,
153
rotary_dim=self.head_dim,
155
- max_position_embeddings=max_position_embeddings,
154
+ max_position=max_position_embeddings,
155
base=self.rope_theta,
156
is_neox_style=True,
158
- dtype=torch.get_default_dtype(),
157
)
158
self.attn = Attention(self.num_heads,
159
self.head_dim,
vllm/model_executor/models/gemma2.py
+4
-6
@@ -32,7 +32,7 @@ from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
32
from vllm.model_executor.layers.logits_processor import LogitsProcessor
33
from vllm.model_executor.layers.quantization.base_config import (
34
QuantizationConfig)
35
-from vllm.model_executor.layers.rotary_embedding import GemmaRotaryEmbedding
35
+from vllm.model_executor.layers.rotary_embedding import get_rope
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding)
@@ -130,14 +130,12 @@ class Gemma2Attention(nn.Module):
130
bias=config.attention_bias,
131
quant_config=quant_config,
132
)
133
- # TODO(woosuk): Use the `get_rope` interface.
134
- self.rotary_emb = GemmaRotaryEmbedding(
133
+ self.rotary_emb = get_rope(
134
self.head_dim,
136
- self.head_dim,
137
- max_position_embeddings,
135
+ rotary_dim=self.head_dim,
136
+ max_position=max_position_embeddings,
137
base=self.rope_theta,
138
is_neox_style=True,
140
- dtype=torch.get_default_dtype(),
139
)
140
141
# FIXME(woosuk): While Gemma 2 uses sliding window attention for every