A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Misc] Clean up RoPE forward_native (#8076)
Woosuk Kwon committed
Sep 4, 2024 at 20:31 UTC
4624d98dbdd6f29a3d8ba7a86d93bde730ef5f7d
1 file changed
+19
-76
vllm/model_executor/layers/rotary_embedding.py
+19
-76
@@ -28,7 +28,6 @@ import torch
28
import torch.nn as nn
29
30
from vllm.model_executor.custom_op import CustomOp
31
-from vllm.platforms import current_platform
31
32
33
def _rotate_neox(x: torch.Tensor) -> torch.Tensor:
@@ -48,21 +47,29 @@ def _apply_rotary_emb(
47
x: torch.Tensor,
48
cos: torch.Tensor,
49
sin: torch.Tensor,
50
+ is_neox_style: bool,
51
) -> torch.Tensor:
52
"""
53
Args:
54
x: [num_tokens, num_heads, head_size]
55
cos: [num_tokens, head_size // 2]
56
sin: [num_tokens, head_size // 2]
57
+ is_neox_style: Whether to use the Neox-style or GPT-J-style rotary
58
+ positional embeddings.
59
"""
58
- orig_dtype = x.dtype
59
- x = x.float()
60
- x1, x2 = torch.chunk(x, 2, dim=-1)
61
- cos = cos.unsqueeze(-2)
62
- sin = sin.unsqueeze(-2)
60
+ cos = cos.unsqueeze(-2).to(x.dtype)
61
+ sin = sin.unsqueeze(-2).to(x.dtype)
62
+ if is_neox_style:
63
+ x1, x2 = torch.chunk(x, 2, dim=-1)
64
+ else:
65
+ x1 = x[..., ::2]
66
+ x2 = x[..., 1::2]
67
o1 = x1 * cos - x2 * sin
68
o2 = x2 * cos + x1 * sin
65
- return torch.cat((o1, o2), dim=-1).to(orig_dtype)
69
+ if is_neox_style:
70
+ return torch.cat((o1, o2), dim=-1)
71
+ else:
72
+ return torch.stack((o1, o2), dim=-1).flatten(-2)
73
74
75
class RotaryEmbedding(CustomOp):
@@ -87,10 +94,9 @@ class RotaryEmbedding(CustomOp):
94
95
cache = self._compute_cos_sin_cache()
96
cache = cache.to(dtype)
97
+ self.cos_sin_cache: torch.Tensor
98
self.register_buffer("cos_sin_cache", cache, persistent=False)
99
92
- self.use_native2 = current_platform.is_tpu() and is_neox_style
93
-
100
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
101
"""Compute the inverse frequency."""
102
# NOTE(woosuk): To exactly match the HF implementation, we need to
@@ -119,59 +125,7 @@ class RotaryEmbedding(CustomOp):
125
key: torch.Tensor,
126
offsets: Optional[torch.Tensor] = None,
127
) -> Tuple[torch.Tensor, torch.Tensor]:
122
- """A PyTorch-native implementation equivalent to forward().
123
-
124
- This method mimics the implementation of the custom CUDA kernel
125
- used in `forward_cuda()`.
126
- """
127
- query = query.view(*query.shape[:-1], -1, self.head_size)
128
- key = key.view(*key.shape[:-1], -1, self.head_size)
129
-
130
- query_rot = query[..., :self.rotary_dim]
131
- key_rot = key[..., :self.rotary_dim]
132
- if self.rotary_dim < self.head_size:
133
- query_pass = query[..., self.rotary_dim:]
134
- key_pass = key[..., self.rotary_dim:]
135
-
136
- self.cos_sin_cache: torch.Tensor = self.cos_sin_cache.to(
137
- positions.device, dtype=query.dtype)
138
- cos_sin = self.cos_sin_cache[torch.add(positions, offsets)
139
- if offsets is not None else positions]
140
- cos, sin = cos_sin.chunk(2, dim=-1)
141
- if self.is_neox_style:
142
- # NOTE(woosuk): Here we assume that the positions tensor has the
143
- # shape [batch_size, seq_len].
144
- cos = cos.repeat(1, 1, 2).unsqueeze(-2)
145
- sin = sin.repeat(1, 1, 2).unsqueeze(-2)
146
- else:
147
- cos = cos.repeat_interleave(2, dim=-1).unsqueeze(-2)
148
- sin = sin.repeat_interleave(2, dim=-1).unsqueeze(-2)
149
-
150
- rotate_fn = _rotate_neox if self.is_neox_style else _rotate_gptj
151
- query_rot = query_rot * cos + rotate_fn(query_rot) * sin
152
- key_rot = key_rot * cos + rotate_fn(key_rot) * sin
153
-
154
- if self.rotary_dim < self.head_size:
155
- query = torch.cat((query_rot, query_pass), dim=-1)
156
- key = torch.cat((key_rot, key_pass), dim=-1)
157
- else:
158
- query = query_rot
159
- key = key_rot
160
- query = query.flatten(-2)
161
- key = key.flatten(-2)
162
- return query, key
163
-
164
- def forward_native2(
165
- self,
166
- positions: torch.Tensor,
167
- query: torch.Tensor,
168
- key: torch.Tensor,
169
- offsets: Optional[torch.Tensor] = None,
170
- ) -> Tuple[torch.Tensor, torch.Tensor]:
171
- """Another PyTorch-native implementation of forward().
172
-
173
- This method might perform better than `forward_native()` when compiled.
174
- """
128
+ """A PyTorch-native implementation of forward()."""
129
if offsets is not None:
130
positions = positions + offsets
131
positions = positions.flatten()
@@ -183,14 +137,14 @@ class RotaryEmbedding(CustomOp):
137
query = query.view(num_tokens, -1, self.head_size)
138
query_rot = query[..., :self.rotary_dim]
139
query_pass = query[..., self.rotary_dim:]
186
- query_rot = _apply_rotary_emb(query_rot, cos, sin)
140
+ query_rot = _apply_rotary_emb(query_rot, cos, sin, self.is_neox_style)
141
query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
142
143
key_shape = key.shape
144
key = key.view(num_tokens, -1, self.head_size)
145
key_rot = key[..., :self.rotary_dim]
146
key_pass = key[..., self.rotary_dim:]
193
- key_rot = _apply_rotary_emb(key_rot, cos, sin)
147
+ key_rot = _apply_rotary_emb(key_rot, cos, sin, self.is_neox_style)
148
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
149
return query, key
150
@@ -203,7 +157,7 @@ class RotaryEmbedding(CustomOp):
157
) -> Tuple[torch.Tensor, torch.Tensor]:
158
from vllm import _custom_ops as ops
159
206
- self.cos_sin_cache = self.cos_sin_cache.to(positions.device,
160
+ self.cos_sin_cache = self.cos_sin_cache.to(query.device,
161
dtype=query.dtype)
162
# ops.rotary_embedding()/batched_rotary_embedding()
163
# are in-place operations that update the query and key tensors.
@@ -240,17 +194,6 @@ class RotaryEmbedding(CustomOp):
194
self.cos_sin_cache, self.is_neox_style)
195
return query, key
196
243
- def forward_tpu(
244
- self,
245
- positions: torch.Tensor,
246
- query: torch.Tensor,
247
- key: torch.Tensor,
248
- offsets: Optional[torch.Tensor] = None,
249
- ) -> Tuple[torch.Tensor, torch.Tensor]:
250
- forward_fn = (self.forward_native2
251
- if self.use_native2 else self.forward_native)
252
- return forward_fn(positions, query, key, offsets)
253
-
197
def extra_repr(self) -> str:
198
s = f"head_size={self.head_size}, rotary_dim={self.rotary_dim}"
199
s += f", max_position_embeddings={self.max_position_embeddings}"