A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Model] Rename Phi3 rope scaling type (#5595)
Amit Garg committed
Jun 17, 2024 at 09:04 UTC
9333fb8eb9ed6a62d33ef4d56d589f83a0f19233
2 files changed
+16
-8
vllm/config.py
+4
-1
@@ -1287,7 +1287,10 @@ def _get_and_verify_max_len(
1287
derived_max_model_len = default_max_len
1288
1289
rope_scaling = getattr(hf_config, "rope_scaling", None)
1290
- if rope_scaling is not None and rope_scaling["type"] != "su":
1290
+ # The correct one should be "longrope", kept "su" here
1291
+ # to be backward compatible
1292
+ if rope_scaling is not None and rope_scaling["type"] != "su" \
1293
+ and rope_scaling["type"] != "longrope":
1294
if disable_sliding_window:
1295
# TODO(robertgshaw): Find a model that supports rope_scaling
1296
# with sliding window to see if this case should be allowed.
vllm/model_executor/layers/rotary_embedding.py
+12
-7
@@ -467,7 +467,7 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
467
return cache
468
469
470
-class Phi3SuScaledRotaryEmbedding(nn.Module):
470
+class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
471
"""Phi3 family of models scaled rotary embedding.
472
473
Based on the original RotaryEmbedding implementation.
@@ -491,11 +491,12 @@ class Phi3SuScaledRotaryEmbedding(nn.Module):
491
492
if rotary_dim != head_size:
493
raise ValueError(
494
- f"`Phi3SuScaledRotaryEmbedding` does not support rotary_dim != \
495
- head_size ({rotary_dim}!={head_size}).")
494
+ f"`Phi3LongRoPEScaledRotaryEmbedding` does not support \
495
+ rotary_dim != head_size ({rotary_dim}!={head_size}).")
496
if is_neox_style is False:
497
raise ValueError(
498
- "`Phi3SuScaledRotaryEmbedding` only supports neox_style.")
498
+ "`Phi3LongRoPEScaledRotaryEmbedding` only supports neox_style."
499
+ )
500
501
self.head_size = head_size
502
self.max_position_embeddings = max_position_embeddings
@@ -608,7 +609,9 @@ def get_rope(
609
is_neox_style, dtype)
610
else:
611
scaling_type = rope_scaling["type"]
611
- if scaling_type != "su":
612
+ # The correct one should be "longrope" but keep "su" here
613
+ # for backward compatible
614
+ if scaling_type != "su" and scaling_type != "longrope":
615
scaling_factor = rope_scaling["factor"]
616
if scaling_type == "linear":
617
rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim,
@@ -633,7 +636,9 @@ def get_rope(
636
base, is_neox_style,
637
scaling_factor, dtype,
638
**extra_kwargs)
636
- elif scaling_type == "su":
639
+ # The correct one should be "longrope" but keep "su" here
640
+ # for backward compatible
641
+ elif scaling_type == "su" or scaling_type == "longrope":
642
short_factor = rope_scaling["short_factor"]
643
long_factor = rope_scaling["long_factor"]
644
original_max_position = rope_scaling[
@@ -643,7 +648,7 @@ def get_rope(
648
for k, v in rope_scaling.items()
649
if k in ("short_mscale", "long_mscale")
650
}
646
- rotary_emb = Phi3SuScaledRotaryEmbedding(
651
+ rotary_emb = Phi3LongRoPEScaledRotaryEmbedding(
652
head_size, rotary_dim, max_position, original_max_position,
653
base, is_neox_style, dtype, short_factor, long_factor,
654
**extra_kwargs)