@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c5df56f88b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

Add support for a rope extension method (#6553)

Simon Mo committed Jul 18, 2024 at 18:53 UTC c5df56f88bc8a5a32a0534793f48182a333aeca4
2 files changed +48 -4
vllm/config.py
+12 -2
@@ -151,6 +151,15 @@ class ModelConfig:
151 self.hf_text_config = get_hf_text_config(self.hf_config)
152 self.dtype = _get_and_verify_dtype(self.hf_text_config, dtype)
153
154 + if (getattr(self.hf_config, "max_position_embeddings", 0) == 131072
155 + and getattr(self.hf_config, "rope_scaling", None) is None):
156 + # Note(simon): this is a special case for a model that doesn't
157 + # supply rope_scaling. We should remove this once the model is
158 + # updated.
159 + self.hf_config.update({"rope_scaling": {
160 + "type": "extended",
161 + }})
162 +
163 if (not self.disable_sliding_window
164 and self.hf_text_config.model_type == "gemma2"
165 and self.hf_text_config.sliding_window is not None):
@@ -1442,8 +1451,9 @@ def _get_and_verify_max_len(
1451 rope_scaling = getattr(hf_config, "rope_scaling", None)
1452 # The correct one should be "longrope", kept "su" here
1453 # to be backward compatible
1445 - if rope_scaling is not None and rope_scaling["type"] != "su" \
1446 - and rope_scaling["type"] != "longrope":
1454 + if rope_scaling is not None and rope_scaling["type"] not in {
1455 + "su", "longrope", "extended"
1456 + }:
1457 if disable_sliding_window:
1458 # TODO(robertgshaw): Find a model that supports rope_scaling
1459 # with sliding window to see if this case should be allowed.
vllm/model_executor/layers/rotary_embedding.py
+36 -2
@@ -733,6 +733,36 @@ class GemmaRotaryEmbedding(RotaryEmbedding):
733 return inv_freq
734
735
736 +class ExtendedRotaryEmbedding(RotaryEmbedding):
737 +
738 + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
739 + inv_freqs = super()._compute_inv_freq(base)
740 + return self.apply_scaling(inv_freqs)
741 +
742 + def apply_scaling(self, freqs: torch.Tensor):
743 + scale_factor = 8
744 + low_freq_factor = 1
745 + high_freq_factor = 4
746 + old_context_len = 8192
747 +
748 + low_freq_wavelen = old_context_len / low_freq_factor
749 + high_freq_wavelen = old_context_len / high_freq_factor
750 + new_freqs = []
751 + for freq in freqs:
752 + wavelen = 2 * math.pi / freq
753 + if wavelen < high_freq_wavelen:
754 + new_freqs.append(freq)
755 + elif wavelen > low_freq_wavelen:
756 + new_freqs.append(freq / scale_factor)
757 + else:
758 + assert low_freq_wavelen != high_freq_wavelen
759 + smooth = (old_context_len / wavelen - low_freq_factor) / (
760 + high_freq_factor - low_freq_factor)
761 + new_freqs.append((1 - smooth) * freq / scale_factor +
762 + smooth * freq)
763 + return torch.tensor(new_freqs, dtype=freqs.dtype, device=freqs.device)
764 +
765 +
766 _ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
767
768
@@ -767,9 +797,13 @@ def get_rope(
797 scaling_type = rope_scaling["type"]
798 # The correct one should be "longrope" but keep "su" here
799 # for backward compatible
770 - if scaling_type != "su" and scaling_type != "longrope":
800 + if scaling_type not in {"su", "longrope", "extended"}:
801 scaling_factor = rope_scaling["factor"]
772 - if scaling_type == "linear":
802 + if scaling_type == "extended":
803 + rotary_emb = ExtendedRotaryEmbedding(head_size, rotary_dim,
804 + max_position, base,
805 + is_neox_style, dtype)
806 + elif scaling_type == "linear":
807 rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim,
808 max_position, base,
809 is_neox_style,