@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9333fb8eb9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Model] Rename Phi3 rope scaling type (#5595)

Amit Garg committed Jun 17, 2024 at 09:04 UTC 9333fb8eb9ed6a62d33ef4d56d589f83a0f19233
2 files changed +16 -8
vllm/config.py
+4 -1
@@ -1287,7 +1287,10 @@ def _get_and_verify_max_len(
1287 derived_max_model_len = default_max_len
1288
1289 rope_scaling = getattr(hf_config, "rope_scaling", None)
1290 - if rope_scaling is not None and rope_scaling["type"] != "su":
1290 + # The correct one should be "longrope", kept "su" here
1291 + # to be backward compatible
1292 + if rope_scaling is not None and rope_scaling["type"] != "su" \
1293 + and rope_scaling["type"] != "longrope":
1294 if disable_sliding_window:
1295 # TODO(robertgshaw): Find a model that supports rope_scaling
1296 # with sliding window to see if this case should be allowed.
vllm/model_executor/layers/rotary_embedding.py
+12 -7
@@ -467,7 +467,7 @@ class YaRNScalingRotaryEmbedding(RotaryEmbedding):
467 return cache
468
469
470 -class Phi3SuScaledRotaryEmbedding(nn.Module):
470 +class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
471 """Phi3 family of models scaled rotary embedding.
472
473 Based on the original RotaryEmbedding implementation.
@@ -491,11 +491,12 @@ class Phi3SuScaledRotaryEmbedding(nn.Module):
491
492 if rotary_dim != head_size:
493 raise ValueError(
494 - f"`Phi3SuScaledRotaryEmbedding` does not support rotary_dim != \
495 - head_size ({rotary_dim}!={head_size}).")
494 + f"`Phi3LongRoPEScaledRotaryEmbedding` does not support \
495 + rotary_dim != head_size ({rotary_dim}!={head_size}).")
496 if is_neox_style is False:
497 raise ValueError(
498 - "`Phi3SuScaledRotaryEmbedding` only supports neox_style.")
498 + "`Phi3LongRoPEScaledRotaryEmbedding` only supports neox_style."
499 + )
500
501 self.head_size = head_size
502 self.max_position_embeddings = max_position_embeddings
@@ -608,7 +609,9 @@ def get_rope(
609 is_neox_style, dtype)
610 else:
611 scaling_type = rope_scaling["type"]
611 - if scaling_type != "su":
612 + # The correct one should be "longrope" but keep "su" here
613 + # for backward compatible
614 + if scaling_type != "su" and scaling_type != "longrope":
615 scaling_factor = rope_scaling["factor"]
616 if scaling_type == "linear":
617 rotary_emb = LinearScalingRotaryEmbedding(head_size, rotary_dim,
@@ -633,7 +636,9 @@ def get_rope(
636 base, is_neox_style,
637 scaling_factor, dtype,
638 **extra_kwargs)
636 - elif scaling_type == "su":
639 + # The correct one should be "longrope" but keep "su" here
640 + # for backward compatible
641 + elif scaling_type == "su" or scaling_type == "longrope":
642 short_factor = rope_scaling["short_factor"]
643 long_factor = rope_scaling["long_factor"]
644 original_max_position = rope_scaling[
@@ -643,7 +648,7 @@ def get_rope(
648 for k, v in rope_scaling.items()
649 if k in ("short_mscale", "long_mscale")
650 }
646 - rotary_emb = Phi3SuScaledRotaryEmbedding(
651 + rotary_emb = Phi3LongRoPEScaledRotaryEmbedding(
652 head_size, rotary_dim, max_position, original_max_position,
653 base, is_neox_style, dtype, short_factor, long_factor,
654 **extra_kwargs)