| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | |
| 5 | import torch |
| 6 | |
| 7 | from .base import RotaryEmbedding |
| 8 | |
| 9 | |
| 10 | class NTKScalingRotaryEmbedding(RotaryEmbedding): |
| 11 | """RotaryEmbedding extended with fixed and mixed NTK scaling. |
| 12 | https://kexue.fm/archives/9706""" |
| 13 | |
| 14 | def __init__( |
| 15 | self, |
| 16 | head_size: int, |
| 17 | rotary_dim: int, |
| 18 | max_position_embeddings: int, |
| 19 | base: float, |
| 20 | is_neox_style: bool, |
| 21 | scaling_factor: float, |
| 22 | dtype: torch.dtype, |
| 23 | mixed_b: float | None = None, |
| 24 | ) -> None: |
| 25 | self.scaling_factor = scaling_factor |
| 26 | self.mixed_b = mixed_b |
| 27 | super().__init__( |
| 28 | head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype |
| 29 | ) |
| 30 | |
| 31 | def _compute_inv_freq(self, base: float) -> torch.Tensor: |
| 32 | base = self.base * (self.scaling_factor if self.mixed_b is None else 1) |
| 33 | inv_freq = super()._compute_inv_freq(base) |
| 34 | |
| 35 | if self.mixed_b is None: |
| 36 | inv_freq = inv_freq / self.scaling_factor ** (2 / self.rotary_dim) |
| 37 | else: |
| 38 | a = ( |
| 39 | torch.tensor(self.scaling_factor).log() |
| 40 | / (self.rotary_dim / 2) ** self.mixed_b |
| 41 | ) |
| 42 | lambda_1_m = ( |
| 43 | a * torch.arange(1, self.rotary_dim // 2 + 1).float() ** self.mixed_b |
| 44 | ).exp() |
| 45 | inv_freq = inv_freq / lambda_1_m |
| 46 | |
| 47 | return inv_freq |