@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 200a2ffa6b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Misc] Refactor Llama3 RoPE initialization (#7637)

Woosuk Kwon committed Aug 18, 2024 at 17:18 UTC 200a2ffa6bc6564ce5a86d48c13a4caae6201cbc
1 file changed +53 -29
vllm/model_executor/layers/rotary_embedding.py
+53 -29
@@ -734,34 +734,50 @@ class GemmaRotaryEmbedding(RotaryEmbedding):
734 return inv_freq
735
736
737 -class ExtendedRotaryEmbedding(RotaryEmbedding):
737 +class Llama3RotaryEmbedding(RotaryEmbedding):
738 +
739 + def __init__(
740 + self,
741 + head_size: int,
742 + rotary_dim: int,
743 + max_position_embeddings: int,
744 + base: int,
745 + is_neox_style: bool,
746 + dtype: torch.dtype,
747 + scaling_factor: float,
748 + low_freq_factor: float,
749 + high_freq_factor: float,
750 + orig_max_position: int,
751 + ) -> None:
752 + self.scaling_factor = scaling_factor
753 + self.low_freq_factor = low_freq_factor
754 + self.high_freq_factor = high_freq_factor
755 + self.orig_max_position = orig_max_position
756 + super().__init__(head_size, rotary_dim, max_position_embeddings, base,
757 + is_neox_style, dtype)
758
759 def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
760 inv_freqs = super()._compute_inv_freq(base)
741 - return self.apply_scaling(inv_freqs)
742 -
743 - def apply_scaling(self, freqs: torch.Tensor):
744 - scale_factor = 8
745 - low_freq_factor = 1
746 - high_freq_factor = 4
747 - old_context_len = 8192
748 -
749 - low_freq_wavelen = old_context_len / low_freq_factor
750 - high_freq_wavelen = old_context_len / high_freq_factor
751 - new_freqs = []
752 - for freq in freqs:
753 - wavelen = 2 * math.pi / freq
754 - if wavelen < high_freq_wavelen:
755 - new_freqs.append(freq)
756 - elif wavelen > low_freq_wavelen:
757 - new_freqs.append(freq / scale_factor)
758 - else:
759 - assert low_freq_wavelen != high_freq_wavelen
760 - smooth = (old_context_len / wavelen - low_freq_factor) / (
761 - high_freq_factor - low_freq_factor)
762 - new_freqs.append((1 - smooth) * freq / scale_factor +
763 - smooth * freq)
764 - return torch.tensor(new_freqs, dtype=freqs.dtype, device=freqs.device)
761 + low_freq_wavelen = self.orig_max_position / self.low_freq_factor
762 + high_freq_wavelen = self.orig_max_position / self.high_freq_factor
763 +
764 + wave_len = 2 * math.pi / inv_freqs
765 + if self.low_freq_factor != self.high_freq_factor:
766 + smooth = (self.orig_max_position / wave_len - self.low_freq_factor
767 + ) / (self.high_freq_factor - self.low_freq_factor)
768 + else:
769 + smooth = 0
770 + new_freqs = torch.where(
771 + wave_len < high_freq_wavelen,
772 + inv_freqs,
773 + torch.where(
774 + wave_len > low_freq_wavelen,
775 + inv_freqs / self.scaling_factor,
776 + (1 - smooth) * inv_freqs / self.scaling_factor +
777 + smooth * inv_freqs,
778 + ),
779 + )
780 + return new_freqs
781
782
783 _ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
@@ -794,6 +810,7 @@ def get_rope(
810 rope_scaling_args, dtype)
811 if key in _ROPE_DICT:
812 return _ROPE_DICT[key]
813 +
814 if rope_scaling is None:
815 rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
816 is_neox_style, dtype)
@@ -802,12 +819,19 @@ def get_rope(
819 "type"] if "type" in rope_scaling else rope_scaling["rope_type"]
820 # The correct one should be "longrope" but keep "su" here
821 # for backward compatible
805 - if scaling_type not in {"su", "longrope", "llama3"}:
822 + if scaling_type not in {"su", "longrope"}:
823 scaling_factor = rope_scaling["factor"]
824 if scaling_type == "llama3":
808 - rotary_emb = ExtendedRotaryEmbedding(head_size, rotary_dim,
809 - max_position, base,
810 - is_neox_style, dtype)
825 + low_freq_factor = rope_scaling["low_freq_factor"]
826 + high_freq_factor = rope_scaling["high_freq_factor"]
827 + original_max_position = rope_scaling[
828 + "original_max_position_embeddings"]
829 + rotary_emb = Llama3RotaryEmbedding(head_size, rotary_dim,
830 + max_position, base,
831 + is_neox_style, dtype,
832 + scaling_factor, low_freq_factor,
833 + high_freq_factor,
834 + original_max_position)
835 elif scaling_type == "linear":
836 rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim,
837 max_position, base,