| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import torch |
| 5 | |
| 6 | from .base import RotaryEmbedding |
| 7 | from .common import yarn_find_correction_range, yarn_get_mscale, yarn_linear_ramp_mask |
| 8 | |
| 9 | |
| 10 | class YaRNScalingRotaryEmbedding(RotaryEmbedding): |
| 11 | """RotaryEmbedding extended with YaRN method. |
| 12 | |
| 13 | Credits to Peng et al. github.com/jquesnelle/yarn |
| 14 | """ |
| 15 | |
| 16 | def __init__( |
| 17 | self, |
| 18 | head_size: int, |
| 19 | rotary_dim: int, |
| 20 | max_position_embeddings: int, |
| 21 | base: float, |
| 22 | is_neox_style: bool, |
| 23 | scaling_factor: float, |
| 24 | dtype: torch.dtype, |
| 25 | *, |
| 26 | beta_fast: int = 32, |
| 27 | beta_slow: int = 1, |
| 28 | mscale: float | None = None, |
| 29 | mscale_all_dim: float | None = None, |
| 30 | attention_factor: float | None = None, |
| 31 | truncate: bool = True, |
| 32 | ) -> None: |
| 33 | self.scaling_factor = scaling_factor |
| 34 | self.beta_fast = beta_fast |
| 35 | self.beta_slow = beta_slow |
| 36 | self.truncate = truncate |
| 37 | # Get n-d magnitude scaling corrected for interpolation |
| 38 | if attention_factor is not None: |
| 39 | self.mscale = float(attention_factor) |
| 40 | elif mscale and mscale_all_dim: |
| 41 | self.mscale = float( |
| 42 | yarn_get_mscale(self.scaling_factor, mscale) |
| 43 | / yarn_get_mscale(self.scaling_factor, mscale_all_dim) |
| 44 | ) |
| 45 | else: |
| 46 | self.mscale = float(yarn_get_mscale(self.scaling_factor)) |
| 47 | super().__init__( |
| 48 | head_size, rotary_dim, max_position_embeddings, base, is_neox_style, dtype |
| 49 | ) |
| 50 | |
| 51 | def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor: |
| 52 | pos_freqs = self.base ** ( |
| 53 | torch.arange(0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim |
| 54 | ) |
| 55 | inv_freq_extrapolation = 1.0 / pos_freqs |
| 56 | inv_freq_interpolation = 1.0 / (scaling_factor * pos_freqs) |
| 57 | |
| 58 | low, high = yarn_find_correction_range( |
| 59 | self.beta_fast, |
| 60 | self.beta_slow, |
| 61 | self.rotary_dim, |
| 62 | self.base, |
| 63 | self.max_position_embeddings, |
| 64 | self.truncate, |
| 65 | ) |
| 66 | # Get n-d rotational scaling corrected for extrapolation |
| 67 | inv_freq_mask = 1 - yarn_linear_ramp_mask( |
| 68 | low, high, self.rotary_dim // 2, dtype=torch.float |
| 69 | ) |
| 70 | inv_freq = ( |
| 71 | inv_freq_interpolation * (1 - inv_freq_mask) |
| 72 | + inv_freq_extrapolation * inv_freq_mask |
| 73 | ) |
| 74 | return inv_freq |
| 75 | |
| 76 | def _compute_cos_sin_cache(self) -> torch.Tensor: |
| 77 | inv_freq = self._compute_inv_freq(self.scaling_factor) |
| 78 | t = torch.arange( |
| 79 | self.max_position_embeddings * self.scaling_factor, dtype=torch.float32 |
| 80 | ) |
| 81 | freqs = torch.einsum("i,j -> ij", t, inv_freq) |
| 82 | cos = freqs.cos() * self.mscale |
| 83 | sin = freqs.sin() * self.mscale |
| 84 | cache = torch.cat((cos, sin), dim=-1) |
| 85 | return cache |