A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[TPU] Optimize RoPE forward_native2 (#7636)
Woosuk Kwon committed
Aug 18, 2024 at 01:15 UTC
ab7165f2c7ea358df969d68a0fb0ce9bb184a083
1 file changed
+27
-26
vllm/model_executor/layers/rotary_embedding.py
+27
-26
@@ -46,15 +46,23 @@ def _rotate_gptj(x: torch.Tensor) -> torch.Tensor:
46
47
def _apply_rotary_emb(
48
x: torch.Tensor,
49
- freqs_cis: torch.Tensor,
49
+ cos: torch.Tensor,
50
+ sin: torch.Tensor,
51
) -> torch.Tensor:
51
- x_ = torch.view_as_complex(
52
- torch.stack(torch.chunk(x.transpose(1, 2).float(), 2, dim=-1), dim=-1))
53
- x_out = torch.view_as_real(x_ * freqs_cis).type_as(x)
54
- x_out = torch.cat(torch.chunk(x_out, 2, dim=-1), dim=-2)
55
- x_out = x_out.reshape(x_out.shape[0], x_out.shape[1], x_out.shape[2],
56
- -1).transpose(1, 2)
57
- return x_out
52
+ """
53
+ Args:
54
+ x: [num_tokens, num_heads, head_size]
55
+ cos: [num_tokens, head_size // 2]
56
+ sin: [num_tokens, head_size // 2]
57
+ """
58
+ orig_dtype = x.dtype
59
+ x = x.float()
60
+ x1, x2 = torch.chunk(x, 2, dim=-1)
61
+ cos = cos.unsqueeze(-2)
62
+ sin = sin.unsqueeze(-2)
63
+ o1 = x1 * cos - x2 * sin
64
+ o2 = x2 * cos + x1 * sin
65
+ return torch.cat((o1, o2), dim=-1).to(orig_dtype)
66
67
68
class RotaryEmbedding(CustomOp):
@@ -78,14 +86,10 @@ class RotaryEmbedding(CustomOp):
86
self.dtype = dtype
87
88
cache = self._compute_cos_sin_cache()
89
+ cache = cache.to(dtype)
90
+ self.register_buffer("cos_sin_cache", cache, persistent=False)
91
+
92
self.use_native2 = current_platform.is_tpu() and is_neox_style
82
- if not self.use_native2:
83
- cache = cache.to(dtype)
84
- self.register_buffer("cos_sin_cache", cache, persistent=False)
85
- else:
86
- cos, sin = cache.chunk(2, dim=-1)
87
- freqs_cis = cos + 1j * sin
88
- self.register_buffer("freqs_cis", freqs_cis, persistent=False)
93
94
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
95
"""Compute the inverse frequency."""
@@ -173,28 +177,25 @@ class RotaryEmbedding(CustomOp):
177
178
This method might perform better than `forward_native()` when compiled.
179
"""
176
- if positions.dim() == 1:
177
- batch_size = 1
178
- seq_len = positions.shape[0]
179
- else:
180
- batch_size, seq_len = positions.shape
180
if offsets is not None:
181
positions = positions + offsets
183
- freqs_cis = self.freqs_cis.index_select(0, positions.flatten())
184
- freqs_cis = freqs_cis.view(batch_size, 1, seq_len, -1)
182
+ positions = positions.flatten()
183
+ num_tokens = positions.shape[0]
184
+ cos_sin = self.cos_sin_cache.index_select(0, positions)
185
+ cos, sin = cos_sin.chunk(2, dim=-1)
186
187
query_shape = query.shape
187
- query = query.view(batch_size, seq_len, -1, self.head_size)
188
+ query = query.view(num_tokens, -1, self.head_size)
189
query_rot = query[..., :self.rotary_dim]
190
query_pass = query[..., self.rotary_dim:]
190
- query_rot = _apply_rotary_emb(query_rot, freqs_cis)
191
+ query_rot = _apply_rotary_emb(query_rot, cos, sin)
192
query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
193
194
key_shape = key.shape
194
- key = key.view(batch_size, seq_len, -1, self.head_size)
195
+ key = key.view(num_tokens, -1, self.head_size)
196
key_rot = key[..., :self.rotary_dim]
197
key_pass = key[..., self.rotary_dim:]
197
- key_rot = _apply_rotary_emb(key_rot, freqs_cis)
198
+ key_rot = _apply_rotary_emb(key_rot, cos, sin)
199
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
200
return query, key
201