A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Misc] Refactor Llama3 RoPE initialization (#7637)
Woosuk Kwon committed
Aug 18, 2024 at 17:18 UTC
200a2ffa6bc6564ce5a86d48c13a4caae6201cbc
1 file changed
+53
-29
vllm/model_executor/layers/rotary_embedding.py
+53
-29
@@ -734,34 +734,50 @@ class GemmaRotaryEmbedding(RotaryEmbedding):
734
return inv_freq
735
736
737
-class ExtendedRotaryEmbedding(RotaryEmbedding):
737
+class Llama3RotaryEmbedding(RotaryEmbedding):
738
+
739
+ def __init__(
740
+ self,
741
+ head_size: int,
742
+ rotary_dim: int,
743
+ max_position_embeddings: int,
744
+ base: int,
745
+ is_neox_style: bool,
746
+ dtype: torch.dtype,
747
+ scaling_factor: float,
748
+ low_freq_factor: float,
749
+ high_freq_factor: float,
750
+ orig_max_position: int,
751
+ ) -> None:
752
+ self.scaling_factor = scaling_factor
753
+ self.low_freq_factor = low_freq_factor
754
+ self.high_freq_factor = high_freq_factor
755
+ self.orig_max_position = orig_max_position
756
+ super().__init__(head_size, rotary_dim, max_position_embeddings, base,
757
+ is_neox_style, dtype)
758
759
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
760
inv_freqs = super()._compute_inv_freq(base)
741
- return self.apply_scaling(inv_freqs)
742
-
743
- def apply_scaling(self, freqs: torch.Tensor):
744
- scale_factor = 8
745
- low_freq_factor = 1
746
- high_freq_factor = 4
747
- old_context_len = 8192
748
-
749
- low_freq_wavelen = old_context_len / low_freq_factor
750
- high_freq_wavelen = old_context_len / high_freq_factor
751
- new_freqs = []
752
- for freq in freqs:
753
- wavelen = 2 * math.pi / freq
754
- if wavelen < high_freq_wavelen:
755
- new_freqs.append(freq)
756
- elif wavelen > low_freq_wavelen:
757
- new_freqs.append(freq / scale_factor)
758
- else:
759
- assert low_freq_wavelen != high_freq_wavelen
760
- smooth = (old_context_len / wavelen - low_freq_factor) / (
761
- high_freq_factor - low_freq_factor)
762
- new_freqs.append((1 - smooth) * freq / scale_factor +
763
- smooth * freq)
764
- return torch.tensor(new_freqs, dtype=freqs.dtype, device=freqs.device)
761
+ low_freq_wavelen = self.orig_max_position / self.low_freq_factor
762
+ high_freq_wavelen = self.orig_max_position / self.high_freq_factor
763
+
764
+ wave_len = 2 * math.pi / inv_freqs
765
+ if self.low_freq_factor != self.high_freq_factor:
766
+ smooth = (self.orig_max_position / wave_len - self.low_freq_factor
767
+ ) / (self.high_freq_factor - self.low_freq_factor)
768
+ else:
769
+ smooth = 0
770
+ new_freqs = torch.where(
771
+ wave_len < high_freq_wavelen,
772
+ inv_freqs,
773
+ torch.where(
774
+ wave_len > low_freq_wavelen,
775
+ inv_freqs / self.scaling_factor,
776
+ (1 - smooth) * inv_freqs / self.scaling_factor +
777
+ smooth * inv_freqs,
778
+ ),
779
+ )
780
+ return new_freqs
781
782
783
_ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
@@ -794,6 +810,7 @@ def get_rope(
810
rope_scaling_args, dtype)
811
if key in _ROPE_DICT:
812
return _ROPE_DICT[key]
813
+
814
if rope_scaling is None:
815
rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
816
is_neox_style, dtype)
@@ -802,12 +819,19 @@ def get_rope(
819
"type"] if "type" in rope_scaling else rope_scaling["rope_type"]
820
# The correct one should be "longrope" but keep "su" here
821
# for backward compatible
805
- if scaling_type not in {"su", "longrope", "llama3"}:
822
+ if scaling_type not in {"su", "longrope"}:
823
scaling_factor = rope_scaling["factor"]
824
if scaling_type == "llama3":
808
- rotary_emb = ExtendedRotaryEmbedding(head_size, rotary_dim,
809
- max_position, base,
810
- is_neox_style, dtype)
825
+ low_freq_factor = rope_scaling["low_freq_factor"]
826
+ high_freq_factor = rope_scaling["high_freq_factor"]
827
+ original_max_position = rope_scaling[
828
+ "original_max_position_embeddings"]
829
+ rotary_emb = Llama3RotaryEmbedding(head_size, rotary_dim,
830
+ max_position, base,
831
+ is_neox_style, dtype,
832
+ scaling_factor, low_freq_factor,
833
+ high_freq_factor,
834
+ original_max_position)
835
elif scaling_type == "linear":
836
rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim,
837
max_position, base,