A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Fix Phi-3 Long RoPE scaling implementation (#5628)
Shukant Pal committed
Jun 18, 2024 at 18:46 UTC
59a1eb59c9cb383e5ea36d7253f81ff2ea7766cc
1 file changed
+14
-4
vllm/model_executor/layers/rotary_embedding.py
+14
-4
@@ -507,8 +507,8 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
507
dtype: torch.dtype,
508
short_factor: List[float],
509
long_factor: List[float],
510
- short_mscale: float = 1.1,
511
- long_mscale: float = 1.225,
510
+ short_mscale: float = 1.0,
511
+ long_mscale: float = 1.0,
512
):
513
super().__init__()
514
@@ -530,6 +530,16 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
530
self.short_mscale = short_mscale
531
self.long_mscale = long_mscale
532
533
+ scale = (self.max_position_embeddings /
534
+ self.original_max_position_embeddings)
535
+
536
+ if scale <= 1.0:
537
+ self.scaling_factor = 1.0
538
+ else:
539
+ self.scaling_factor = math.sqrt(
540
+ 1 + math.log(scale) /
541
+ math.log(self.original_max_position_embeddings))
542
+
543
short_cache = self._compute_cos_sin_cache(
544
original_max_position_embeddings, short_factor, short_mscale)
545
short_cache = short_cache.to(dtype)
@@ -565,8 +575,8 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
575
inv_freq = self._compute_inv_freq(rescale_factors)
576
t = torch.arange(max_position_embeddings, dtype=torch.float)
577
freqs = torch.einsum("i,j -> ij", t, inv_freq)
568
- cos = freqs.cos() * mscale
569
- sin = freqs.sin() * mscale
578
+ cos = freqs.cos() * mscale * self.scaling_factor
579
+ sin = freqs.sin() * mscale * self.scaling_factor
580
cache = torch.cat((cos, sin), dim=-1)
581
return cache
582