@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 59a1eb59c9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Fix Phi-3 Long RoPE scaling implementation (#5628)

Shukant Pal committed Jun 18, 2024 at 18:46 UTC 59a1eb59c9cb383e5ea36d7253f81ff2ea7766cc
1 file changed +14 -4
vllm/model_executor/layers/rotary_embedding.py
+14 -4
@@ -507,8 +507,8 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
507 dtype: torch.dtype,
508 short_factor: List[float],
509 long_factor: List[float],
510 - short_mscale: float = 1.1,
511 - long_mscale: float = 1.225,
510 + short_mscale: float = 1.0,
511 + long_mscale: float = 1.0,
512 ):
513 super().__init__()
514
@@ -530,6 +530,16 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
530 self.short_mscale = short_mscale
531 self.long_mscale = long_mscale
532
533 + scale = (self.max_position_embeddings /
534 + self.original_max_position_embeddings)
535 +
536 + if scale <= 1.0:
537 + self.scaling_factor = 1.0
538 + else:
539 + self.scaling_factor = math.sqrt(
540 + 1 + math.log(scale) /
541 + math.log(self.original_max_position_embeddings))
542 +
543 short_cache = self._compute_cos_sin_cache(
544 original_max_position_embeddings, short_factor, short_mscale)
545 short_cache = short_cache.to(dtype)
@@ -565,8 +575,8 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
575 inv_freq = self._compute_inv_freq(rescale_factors)
576 t = torch.arange(max_position_embeddings, dtype=torch.float)
577 freqs = torch.einsum("i,j -> ij", t, inv_freq)
568 - cos = freqs.cos() * mscale
569 - sin = freqs.sin() * mscale
578 + cos = freqs.cos() * mscale * self.scaling_factor
579 + sin = freqs.sin() * mscale * self.scaling_factor
580 cache = torch.cat((cos, sin), dim=-1)
581 return cache
582