@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 538fab93cd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

PR #12718 (#12718)

Amit Garg committed Feb 7, 2025 at 06:22 UTC 538fab93cdd36e965ea1888143dab0df57c8ba84
2 files changed +15 -8
vllm/model_executor/layers/rotary_embedding.py
+11 -7
@@ -509,15 +509,12 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
509 ):
510 super().__init__()
511
512 - if rotary_dim != head_size:
513 - raise ValueError(
514 - f"`Phi3LongRoPEScaledRotaryEmbedding` does not support \
515 - rotary_dim != head_size ({rotary_dim}!={head_size}).")
512 if is_neox_style is False:
513 raise ValueError(
514 "`Phi3LongRoPEScaledRotaryEmbedding` only supports neox_style."
515 )
516
517 + self.rotary_dim = rotary_dim
518 self.head_size = head_size
519 self.max_position_embeddings = max_position_embeddings
520 self.original_max_position_embeddings = original_max_position_embeddings
@@ -557,7 +554,7 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
554 def _compute_inv_freq(self, rescale_factors: List[float]) -> torch.Tensor:
555 rescale_factors = torch.tensor(rescale_factors, dtype=torch.float32)
556 inv_freq = 1.0 / (rescale_factors * (self.base**(torch.arange(
560 - 0, self.head_size, 2, dtype=torch.float) / self.head_size)))
557 + 0, self.rotary_dim, 2, dtype=torch.float) / self.rotary_dim)))
558 return inv_freq
559
560 def _compute_cos_sin_cache(
@@ -596,8 +593,15 @@ class Phi3LongRoPEScaledRotaryEmbedding(nn.Module):
593 cos = cos.repeat(1, 2).unsqueeze(-2)
594 sin = sin.repeat(1, 2).unsqueeze(-2)
595
599 - query = query * cos + _rotate_neox(query) * sin
600 - key = key * cos + _rotate_neox(key) * sin
596 + query_rot = query[..., :self.rotary_dim]
597 + query_pass = query[..., self.rotary_dim:]
598 + query_rot = query_rot * cos + _rotate_neox(query_rot) * sin
599 + query = torch.cat((query_rot, query_pass), dim=-1)
600 +
601 + key_rot = key[..., :self.rotary_dim]
602 + key_pass = key[..., self.rotary_dim:]
603 + key_rot = key_rot * cos + _rotate_neox(key_rot) * sin
604 + key = torch.cat((key_rot, key_pass), dim=-1)
605
606 return query.flatten(-2), key.flatten(-2)
607
vllm/model_executor/models/llama.py
+4 -1
@@ -128,6 +128,9 @@ class LlamaAttention(nn.Module):
128 # MistralConfig has an optional head_dim introduced by Mistral-Nemo
129 self.head_dim = getattr(config, "head_dim",
130 self.hidden_size // self.total_num_heads)
131 + # Phi models introduced a partial_rotary_factor parameter in the config
132 + partial_rotary_factor = getattr(config, "partial_rotary_factor", 1)
133 + self.rotary_dim = int(partial_rotary_factor * self.head_dim)
134 self.q_size = self.num_heads * self.head_dim
135 self.kv_size = self.num_kv_heads * self.head_dim
136 self.scaling = self.head_dim**-0.5
@@ -159,7 +162,7 @@ class LlamaAttention(nn.Module):
162
163 self.rotary_emb = get_rope(
164 self.head_dim,
162 - rotary_dim=self.head_dim,
165 + rotary_dim=self.rotary_dim,
166 max_position=max_position_embeddings,
167 base=rope_theta,
168 rope_scaling=rope_scaling,