A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[BugFix] Implement RoPE for GPT-J (#941)
Woosuk Kwon committed
Sep 6, 2023 at 11:54 UTC
320a622ec4d098f2da5d097930f4031517e7327b
5 files changed
+122
-72
csrc/pos_encoding.cpp
+6
-5
@@ -1,15 +1,16 @@
1
#include <torch/extension.h>
2
3
-void rotary_embedding_neox(
3
+void rotary_embedding(
4
torch::Tensor& positions,
5
torch::Tensor& query,
6
torch::Tensor& key,
7
int head_size,
8
- torch::Tensor& cos_sin_cache);
8
+ torch::Tensor& cos_sin_cache,
9
+ bool is_neox);
10
11
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
12
m.def(
12
- "rotary_embedding_neox",
13
- &rotary_embedding_neox,
14
- "Apply GPT-NeoX style rotary embedding to query and key");
13
+ "rotary_embedding",
14
+ &rotary_embedding,
15
+ "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
16
}
csrc/pos_encoding_kernels.cu
+68
-45
@@ -5,8 +5,38 @@
5
6
namespace vllm {
7
8
-template<typename scalar_t>
9
-__global__ void rotary_embedding_neox_kernel(
8
+template<typename scalar_t, bool IS_NEOX>
9
+inline __device__ void apply_rotary_embedding(
10
+ scalar_t* __restrict__ arr,
11
+ const scalar_t* __restrict__ cos_ptr,
12
+ const scalar_t* __restrict__ sin_ptr,
13
+ int rot_offset,
14
+ int embed_dim)
15
+{
16
+ int x_index, y_index;
17
+ scalar_t cos, sin;
18
+ if (IS_NEOX) {
19
+ // GPT-NeoX style rotary embedding.
20
+ x_index = rot_offset;
21
+ y_index = embed_dim + rot_offset;
22
+ cos = __ldg(cos_ptr + x_index);
23
+ sin = __ldg(sin_ptr + x_index);
24
+ } else {
25
+ // GPT-J style rotary embedding.
26
+ x_index = 2 * rot_offset;
27
+ y_index = 2 * rot_offset + 1;
28
+ cos = __ldg(cos_ptr + x_index / 2);
29
+ sin = __ldg(sin_ptr + x_index / 2);
30
+ }
31
+
32
+ const scalar_t x = arr[x_index];
33
+ const scalar_t y = arr[y_index];
34
+ arr[x_index] = x * cos - y * sin;
35
+ arr[y_index] = y * cos + x * sin;
36
+}
37
+
38
+template<typename scalar_t, bool IS_NEOX>
39
+__global__ void rotary_embedding_kernel(
40
const int64_t* __restrict__ positions, // [num_tokens]
41
scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
42
scalar_t* __restrict__ key, // [num_tokens, num_kv_heads, head_size]
@@ -23,58 +53,37 @@ __global__ void rotary_embedding_neox_kernel(
53
const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
54
55
const int embed_dim = rot_dim / 2;
56
+ const scalar_t* cos_ptr = cache_ptr;
57
+ const scalar_t* sin_ptr = cache_ptr + embed_dim;
58
+
59
const int nq = num_heads * embed_dim;
60
for (int i = threadIdx.x; i < nq; i += blockDim.x) {
61
const int head_idx = i / embed_dim;
62
const int token_head = token_idx * query_stride + head_idx * head_size;
30
-
63
const int rot_offset = i % embed_dim;
32
- const int x_index = rot_offset;
33
- const int y_index = embed_dim + rot_offset;
34
-
35
- const int out_x = token_idx * query_stride + head_idx * head_size + x_index;
36
- const int out_y = token_idx * query_stride + head_idx * head_size + y_index;
37
-
38
- const scalar_t cos = __ldg(cache_ptr + x_index);
39
- const scalar_t sin = __ldg(cache_ptr + y_index);
40
-
41
- const scalar_t q_x = query[token_head + x_index];
42
- const scalar_t q_y = query[token_head + y_index];
43
- query[out_x] = q_x * cos - q_y * sin;
44
- query[out_y] = q_y * cos + q_x * sin;
64
+ apply_rotary_embedding<scalar_t, IS_NEOX>(query + token_head, cos_ptr,
65
+ sin_ptr, rot_offset, embed_dim);
66
}
67
68
const int nk = num_kv_heads * embed_dim;
69
for (int i = threadIdx.x; i < nk; i += blockDim.x) {
70
const int head_idx = i / embed_dim;
71
const int token_head = token_idx * key_stride + head_idx * head_size;
51
-
72
const int rot_offset = i % embed_dim;
53
- const int x_index = rot_offset;
54
- const int y_index = embed_dim + rot_offset;
55
-
56
- const int out_x = token_idx * key_stride + head_idx * head_size + x_index;
57
- const int out_y = token_idx * key_stride + head_idx * head_size + y_index;
58
-
59
- const scalar_t cos = __ldg(cache_ptr + x_index);
60
- const scalar_t sin = __ldg(cache_ptr + y_index);
61
-
62
- const scalar_t k_x = key[token_head + x_index];
63
- const scalar_t k_y = key[token_head + y_index];
64
- key[out_x] = k_x * cos - k_y * sin;
65
- key[out_y] = k_y * cos + k_x * sin;
73
+ apply_rotary_embedding<scalar_t, IS_NEOX>(key + token_head, cos_ptr,
74
+ sin_ptr, rot_offset, embed_dim);
75
}
76
}
77
78
} // namespace vllm
79
71
-void rotary_embedding_neox(
80
+void rotary_embedding(
81
torch::Tensor& positions, // [num_tokens]
82
torch::Tensor& query, // [num_tokens, num_heads * head_size]
83
torch::Tensor& key, // [num_tokens, num_kv_heads * head_size]
84
int head_size,
76
- torch::Tensor& cos_sin_cache) // [max_position, rot_dim]
77
-{
85
+ torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
86
+ bool is_neox) {
87
int num_tokens = query.size(0);
88
int rot_dim = cos_sin_cache.size(1);
89
int num_heads = query.size(1) / head_size;
@@ -87,18 +96,32 @@ void rotary_embedding_neox(
96
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
97
VLLM_DISPATCH_FLOATING_TYPES(
98
query.scalar_type(),
90
- "rotary_embedding_neox",
99
+ "rotary_embedding",
100
[&] {
92
- vllm::rotary_embedding_neox_kernel<scalar_t><<<grid, block, 0, stream>>>(
93
- positions.data_ptr<int64_t>(),
94
- query.data_ptr<scalar_t>(),
95
- key.data_ptr<scalar_t>(),
96
- cos_sin_cache.data_ptr<scalar_t>(),
97
- rot_dim,
98
- query_stride,
99
- key_stride,
100
- num_heads,
101
- num_kv_heads,
102
- head_size);
101
+ if (is_neox) {
102
+ vllm::rotary_embedding_kernel<scalar_t, true><<<grid, block, 0, stream>>>(
103
+ positions.data_ptr<int64_t>(),
104
+ query.data_ptr<scalar_t>(),
105
+ key.data_ptr<scalar_t>(),
106
+ cos_sin_cache.data_ptr<scalar_t>(),
107
+ rot_dim,
108
+ query_stride,
109
+ key_stride,
110
+ num_heads,
111
+ num_kv_heads,
112
+ head_size);
113
+ } else {
114
+ vllm::rotary_embedding_kernel<scalar_t, false><<<grid, block, 0, stream>>>(
115
+ positions.data_ptr<int64_t>(),
116
+ query.data_ptr<scalar_t>(),
117
+ key.data_ptr<scalar_t>(),
118
+ cos_sin_cache.data_ptr<scalar_t>(),
119
+ rot_dim,
120
+ query_stride,
121
+ key_stride,
122
+ num_heads,
123
+ num_kv_heads,
124
+ head_size);
125
+ }
126
});
127
}
tests/kernels/test_pos_encoding.py
+38
-18
@@ -7,49 +7,64 @@ import torch.nn.functional as F
7
8
from vllm import pos_encoding_ops
9
10
+IS_NEOX_STYLE = [True, False]
11
DTYPES = [torch.half, torch.bfloat16, torch.float]
12
HEAD_SIZES = [64, 80, 96, 112, 128, 256]
13
ROTARY_DIMS = [None, 32] # None means rotary dim == head size
14
NUM_HEADS = [7, 12, 40, 52] # Arbitrary values for testing
14
-NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
15
+NUM_TOKENS = [11, 83, 2048] # Arbitrary values for testing
16
SEEDS = [0]
17
18
18
-def rotate_half(x: torch.Tensor) -> torch.Tensor:
19
+def rotate_neox(x: torch.Tensor) -> torch.Tensor:
20
x1 = x[..., :x.shape[-1] // 2]
21
x2 = x[..., x.shape[-1] // 2:]
22
return torch.cat((-x2, x1), dim=-1)
23
24
24
-def apply_rotary_pos_emb(
25
+def rotate_gptj(x: torch.Tensor) -> torch.Tensor:
26
+ x1 = x[..., ::2]
27
+ x2 = x[..., 1::2]
28
+ x = torch.stack((-x2, x1), dim=-1)
29
+ return x.flatten(-2)
30
+
31
+
32
+def apply_rope(
33
q: torch.Tensor,
34
k: torch.Tensor,
35
cos: torch.Tensor,
36
sin: torch.Tensor,
37
+ is_neox_style: bool,
38
) -> Tuple[torch.Tensor, torch.Tensor]:
30
- q_embed = (q * cos) + (rotate_half(q) * sin)
31
- k_embed = (k * cos) + (rotate_half(k) * sin)
39
+ rotate_fn = rotate_neox if is_neox_style else rotate_gptj
40
+ q_embed = (q * cos) + (rotate_fn(q) * sin)
41
+ k_embed = (k * cos) + (rotate_fn(k) * sin)
42
return q_embed, k_embed
43
44
35
-class RefRotaryEmbeddingNeox(nn.Module):
36
- """Reference implementation of the GPT-NeoX style rotary embedding."""
45
+class RefRotaryEmbedding(nn.Module):
46
+ """Reference implementation of rotary embedding."""
47
48
def __init__(
49
self,
50
dim: int,
41
- max_position_embeddings: int = 2048,
51
+ is_neox_style: bool,
52
+ max_position_embeddings: int = 8192,
53
base: int = 10000,
54
) -> None:
55
super().__init__()
56
self.rotary_dim = dim
57
+ self.is_neox_style = is_neox_style
58
self.max_position_embeddings = max_position_embeddings
59
60
# Create cos and sin embeddings.
61
inv_freq = 1.0 / (base**(torch.arange(0, dim, 2) / dim))
62
t = torch.arange(max_position_embeddings).float()
63
freqs = torch.einsum("i,j->ij", t, inv_freq.float())
52
- emb = torch.cat((freqs, freqs), dim=-1)
64
+ if is_neox_style:
65
+ emb = torch.cat((freqs, freqs), dim=-1)
66
+ else:
67
+ emb = torch.repeat_interleave(freqs, 2, -1)
68
cos = emb.cos().to(dtype=inv_freq.dtype)
69
sin = emb.sin().to(dtype=inv_freq.dtype)
70
self.register_buffer("cos_cached", cos, persistent=False)
@@ -61,7 +76,6 @@ class RefRotaryEmbeddingNeox(nn.Module):
76
query: torch.Tensor, # [num_tokens, num_heads, head_size]
77
key: torch.Tensor, # [num_tokens, num_heads, head_size]
78
) -> Tuple[torch.Tensor, torch.Tensor]:
64
-
79
query_rot = query[..., :self.rotary_dim]
80
query_pass = query[..., self.rotary_dim:]
81
key_rot = key[..., :self.rotary_dim]
@@ -71,7 +85,9 @@ class RefRotaryEmbeddingNeox(nn.Module):
85
key_rot = key_rot.transpose(0, 1)
86
cos = F.embedding(positions, self.cos_cached)
87
sin = F.embedding(positions, self.sin_cached)
74
- query_rot, key_rot = apply_rotary_pos_emb(query_rot, key_rot, cos, sin)
88
+
89
+ query_rot, key_rot = apply_rope(query_rot, key_rot, cos, sin,
90
+ self.is_neox_style)
91
query_rot = query_rot.transpose(0, 1).contiguous()
92
key_rot = key_rot.transpose(0, 1).contiguous()
93
@@ -82,6 +98,7 @@ class RefRotaryEmbeddingNeox(nn.Module):
98
return query, key
99
100
101
+@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
102
@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
103
@pytest.mark.parametrize("num_heads", NUM_HEADS)
104
@pytest.mark.parametrize("head_size", HEAD_SIZES)
@@ -89,7 +106,8 @@ class RefRotaryEmbeddingNeox(nn.Module):
106
@pytest.mark.parametrize("dtype", DTYPES)
107
@pytest.mark.parametrize("seed", SEEDS)
108
@torch.inference_mode()
92
-def test_rotary_embedding_neox(
109
+def test_rotary_embedding(
110
+ is_neox_style: bool,
111
num_tokens: int,
112
num_heads: int,
113
head_size: int,
@@ -104,15 +122,15 @@ def test_rotary_embedding_neox(
122
torch.random.manual_seed(seed)
123
torch.cuda.manual_seed(seed)
124
107
- positions = torch.randint(0, max_position, (num_tokens, ), device='cuda')
125
+ positions = torch.randint(0, max_position, (num_tokens, ), device="cuda")
126
query = torch.randn(num_tokens,
127
num_heads * head_size,
128
dtype=dtype,
111
- device='cuda')
129
+ device="cuda")
130
key = torch.randn(num_tokens,
131
num_heads * head_size,
132
dtype=dtype,
115
- device='cuda')
133
+ device="cuda")
134
135
# Create the rotary embedding.
136
inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
@@ -126,20 +144,22 @@ def test_rotary_embedding_neox(
144
# Run the kernel. The kernel is in-place, so we need to clone the inputs.
145
out_query = query.clone()
146
out_key = key.clone()
129
- pos_encoding_ops.rotary_embedding_neox(
147
+ pos_encoding_ops.rotary_embedding(
148
positions,
149
out_query,
150
out_key,
151
head_size,
152
cos_sin_cache,
153
+ is_neox_style,
154
)
155
156
# Run the reference implementation.
138
- ref_rotary_embedding = RefRotaryEmbeddingNeox(
157
+ ref_rotary_embedding = RefRotaryEmbedding(
158
dim=rotary_dim,
159
+ is_neox_style=is_neox_style,
160
max_position_embeddings=max_position,
161
base=base,
142
- ).to(dtype=dtype, device='cuda')
162
+ ).to(dtype=dtype, device="cuda")
163
ref_query, ref_key = ref_rotary_embedding(
164
positions,
165
query.view(num_tokens, num_heads, head_size),
vllm/model_executor/layers/attention.py
+5
-2
@@ -242,7 +242,7 @@ class PagedAttention(nn.Module):
242
243
244
class PagedAttentionWithRoPE(PagedAttention):
245
- """PagedAttention with GPT-NeoX style rotary embedding."""
245
+ """PagedAttention with rotary embedding."""
246
247
def __init__(
248
self,
@@ -253,8 +253,10 @@ class PagedAttentionWithRoPE(PagedAttention):
253
max_position: int = 8192,
254
base: int = 10000,
255
num_kv_heads: Optional[int] = None,
256
+ is_neox_style: bool = True,
257
) -> None:
258
super().__init__(num_heads, head_size, scale, num_kv_heads)
259
+ self.is_neox_style = is_neox_style
260
261
# Create the cos and sin cache.
262
inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
@@ -303,12 +305,13 @@ class PagedAttentionWithRoPE(PagedAttention):
305
306
# Apply rotary embedding to the query and key before passing them
307
# to the attention op.
306
- pos_encoding_ops.rotary_embedding_neox(
308
+ pos_encoding_ops.rotary_embedding(
309
positions,
310
query,
311
key,
312
self.head_size,
313
self.cos_sin_cache,
314
+ self.is_neox_style,
315
)
316
return super().forward(
317
query,
vllm/model_executor/models/gpt_j.py
+5
-2
@@ -67,8 +67,11 @@ class GPTJAttention(nn.Module):
67
scaling = self.head_size**-0.5
68
assert getattr(config, "rotary", True)
69
assert config.rotary_dim % 2 == 0
70
- self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_size,
71
- scaling, config.rotary_dim)
70
+ self.attn = PagedAttentionWithRoPE(self.num_heads,
71
+ self.head_size,
72
+ scaling,
73
+ config.rotary_dim,
74
+ is_neox_style=False)
75
self.warmup = False
76
77
def forward(