@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ab7165f2c7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[TPU] Optimize RoPE forward_native2 (#7636)

Woosuk Kwon committed Aug 18, 2024 at 01:15 UTC ab7165f2c7ea358df969d68a0fb0ce9bb184a083
1 file changed +27 -26
vllm/model_executor/layers/rotary_embedding.py
+27 -26
@@ -46,15 +46,23 @@ def _rotate_gptj(x: torch.Tensor) -> torch.Tensor:
46
47 def _apply_rotary_emb(
48 x: torch.Tensor,
49 - freqs_cis: torch.Tensor,
49 + cos: torch.Tensor,
50 + sin: torch.Tensor,
51 ) -> torch.Tensor:
51 - x_ = torch.view_as_complex(
52 - torch.stack(torch.chunk(x.transpose(1, 2).float(), 2, dim=-1), dim=-1))
53 - x_out = torch.view_as_real(x_ * freqs_cis).type_as(x)
54 - x_out = torch.cat(torch.chunk(x_out, 2, dim=-1), dim=-2)
55 - x_out = x_out.reshape(x_out.shape[0], x_out.shape[1], x_out.shape[2],
56 - -1).transpose(1, 2)
57 - return x_out
52 + """
53 + Args:
54 + x: [num_tokens, num_heads, head_size]
55 + cos: [num_tokens, head_size // 2]
56 + sin: [num_tokens, head_size // 2]
57 + """
58 + orig_dtype = x.dtype
59 + x = x.float()
60 + x1, x2 = torch.chunk(x, 2, dim=-1)
61 + cos = cos.unsqueeze(-2)
62 + sin = sin.unsqueeze(-2)
63 + o1 = x1 * cos - x2 * sin
64 + o2 = x2 * cos + x1 * sin
65 + return torch.cat((o1, o2), dim=-1).to(orig_dtype)
66
67
68 class RotaryEmbedding(CustomOp):
@@ -78,14 +86,10 @@ class RotaryEmbedding(CustomOp):
86 self.dtype = dtype
87
88 cache = self._compute_cos_sin_cache()
89 + cache = cache.to(dtype)
90 + self.register_buffer("cos_sin_cache", cache, persistent=False)
91 +
92 self.use_native2 = current_platform.is_tpu() and is_neox_style
82 - if not self.use_native2:
83 - cache = cache.to(dtype)
84 - self.register_buffer("cos_sin_cache", cache, persistent=False)
85 - else:
86 - cos, sin = cache.chunk(2, dim=-1)
87 - freqs_cis = cos + 1j * sin
88 - self.register_buffer("freqs_cis", freqs_cis, persistent=False)
93
94 def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
95 """Compute the inverse frequency."""
@@ -173,28 +177,25 @@ class RotaryEmbedding(CustomOp):
177
178 This method might perform better than `forward_native()` when compiled.
179 """
176 - if positions.dim() == 1:
177 - batch_size = 1
178 - seq_len = positions.shape[0]
179 - else:
180 - batch_size, seq_len = positions.shape
180 if offsets is not None:
181 positions = positions + offsets
183 - freqs_cis = self.freqs_cis.index_select(0, positions.flatten())
184 - freqs_cis = freqs_cis.view(batch_size, 1, seq_len, -1)
182 + positions = positions.flatten()
183 + num_tokens = positions.shape[0]
184 + cos_sin = self.cos_sin_cache.index_select(0, positions)
185 + cos, sin = cos_sin.chunk(2, dim=-1)
186
187 query_shape = query.shape
187 - query = query.view(batch_size, seq_len, -1, self.head_size)
188 + query = query.view(num_tokens, -1, self.head_size)
189 query_rot = query[..., :self.rotary_dim]
190 query_pass = query[..., self.rotary_dim:]
190 - query_rot = _apply_rotary_emb(query_rot, freqs_cis)
191 + query_rot = _apply_rotary_emb(query_rot, cos, sin)
192 query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
193
194 key_shape = key.shape
194 - key = key.view(batch_size, seq_len, -1, self.head_size)
195 + key = key.view(num_tokens, -1, self.head_size)
196 key_rot = key[..., :self.rotary_dim]
197 key_pass = key[..., self.rotary_dim:]
197 - key_rot = _apply_rotary_emb(key_rot, freqs_cis)
198 + key_rot = _apply_rotary_emb(key_rot, cos, sin)
199 key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
200 return query, key
201