@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 320a622ec4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[BugFix] Implement RoPE for GPT-J (#941)

Woosuk Kwon committed Sep 6, 2023 at 11:54 UTC 320a622ec4d098f2da5d097930f4031517e7327b
5 files changed +122 -72
csrc/pos_encoding.cpp
+6 -5
@@ -1,15 +1,16 @@
1 #include <torch/extension.h>
2
3 -void rotary_embedding_neox(
3 +void rotary_embedding(
4 torch::Tensor& positions,
5 torch::Tensor& query,
6 torch::Tensor& key,
7 int head_size,
8 - torch::Tensor& cos_sin_cache);
8 + torch::Tensor& cos_sin_cache,
9 + bool is_neox);
10
11 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
12 m.def(
12 - "rotary_embedding_neox",
13 - &rotary_embedding_neox,
14 - "Apply GPT-NeoX style rotary embedding to query and key");
13 + "rotary_embedding",
14 + &rotary_embedding,
15 + "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
16 }
csrc/pos_encoding_kernels.cu
+68 -45
@@ -5,8 +5,38 @@
5
6 namespace vllm {
7
8 -template<typename scalar_t>
9 -__global__ void rotary_embedding_neox_kernel(
8 +template<typename scalar_t, bool IS_NEOX>
9 +inline __device__ void apply_rotary_embedding(
10 + scalar_t* __restrict__ arr,
11 + const scalar_t* __restrict__ cos_ptr,
12 + const scalar_t* __restrict__ sin_ptr,
13 + int rot_offset,
14 + int embed_dim)
15 +{
16 + int x_index, y_index;
17 + scalar_t cos, sin;
18 + if (IS_NEOX) {
19 + // GPT-NeoX style rotary embedding.
20 + x_index = rot_offset;
21 + y_index = embed_dim + rot_offset;
22 + cos = __ldg(cos_ptr + x_index);
23 + sin = __ldg(sin_ptr + x_index);
24 + } else {
25 + // GPT-J style rotary embedding.
26 + x_index = 2 * rot_offset;
27 + y_index = 2 * rot_offset + 1;
28 + cos = __ldg(cos_ptr + x_index / 2);
29 + sin = __ldg(sin_ptr + x_index / 2);
30 + }
31 +
32 + const scalar_t x = arr[x_index];
33 + const scalar_t y = arr[y_index];
34 + arr[x_index] = x * cos - y * sin;
35 + arr[y_index] = y * cos + x * sin;
36 +}
37 +
38 +template<typename scalar_t, bool IS_NEOX>
39 +__global__ void rotary_embedding_kernel(
40 const int64_t* __restrict__ positions, // [num_tokens]
41 scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
42 scalar_t* __restrict__ key, // [num_tokens, num_kv_heads, head_size]
@@ -23,58 +53,37 @@ __global__ void rotary_embedding_neox_kernel(
53 const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
54
55 const int embed_dim = rot_dim / 2;
56 + const scalar_t* cos_ptr = cache_ptr;
57 + const scalar_t* sin_ptr = cache_ptr + embed_dim;
58 +
59 const int nq = num_heads * embed_dim;
60 for (int i = threadIdx.x; i < nq; i += blockDim.x) {
61 const int head_idx = i / embed_dim;
62 const int token_head = token_idx * query_stride + head_idx * head_size;
30 -
63 const int rot_offset = i % embed_dim;
32 - const int x_index = rot_offset;
33 - const int y_index = embed_dim + rot_offset;
34 -
35 - const int out_x = token_idx * query_stride + head_idx * head_size + x_index;
36 - const int out_y = token_idx * query_stride + head_idx * head_size + y_index;
37 -
38 - const scalar_t cos = __ldg(cache_ptr + x_index);
39 - const scalar_t sin = __ldg(cache_ptr + y_index);
40 -
41 - const scalar_t q_x = query[token_head + x_index];
42 - const scalar_t q_y = query[token_head + y_index];
43 - query[out_x] = q_x * cos - q_y * sin;
44 - query[out_y] = q_y * cos + q_x * sin;
64 + apply_rotary_embedding<scalar_t, IS_NEOX>(query + token_head, cos_ptr,
65 + sin_ptr, rot_offset, embed_dim);
66 }
67
68 const int nk = num_kv_heads * embed_dim;
69 for (int i = threadIdx.x; i < nk; i += blockDim.x) {
70 const int head_idx = i / embed_dim;
71 const int token_head = token_idx * key_stride + head_idx * head_size;
51 -
72 const int rot_offset = i % embed_dim;
53 - const int x_index = rot_offset;
54 - const int y_index = embed_dim + rot_offset;
55 -
56 - const int out_x = token_idx * key_stride + head_idx * head_size + x_index;
57 - const int out_y = token_idx * key_stride + head_idx * head_size + y_index;
58 -
59 - const scalar_t cos = __ldg(cache_ptr + x_index);
60 - const scalar_t sin = __ldg(cache_ptr + y_index);
61 -
62 - const scalar_t k_x = key[token_head + x_index];
63 - const scalar_t k_y = key[token_head + y_index];
64 - key[out_x] = k_x * cos - k_y * sin;
65 - key[out_y] = k_y * cos + k_x * sin;
73 + apply_rotary_embedding<scalar_t, IS_NEOX>(key + token_head, cos_ptr,
74 + sin_ptr, rot_offset, embed_dim);
75 }
76 }
77
78 } // namespace vllm
79
71 -void rotary_embedding_neox(
80 +void rotary_embedding(
81 torch::Tensor& positions, // [num_tokens]
82 torch::Tensor& query, // [num_tokens, num_heads * head_size]
83 torch::Tensor& key, // [num_tokens, num_kv_heads * head_size]
84 int head_size,
76 - torch::Tensor& cos_sin_cache) // [max_position, rot_dim]
77 -{
85 + torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
86 + bool is_neox) {
87 int num_tokens = query.size(0);
88 int rot_dim = cos_sin_cache.size(1);
89 int num_heads = query.size(1) / head_size;
@@ -87,18 +96,32 @@ void rotary_embedding_neox(
96 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
97 VLLM_DISPATCH_FLOATING_TYPES(
98 query.scalar_type(),
90 - "rotary_embedding_neox",
99 + "rotary_embedding",
100 [&] {
92 - vllm::rotary_embedding_neox_kernel<scalar_t><<<grid, block, 0, stream>>>(
93 - positions.data_ptr<int64_t>(),
94 - query.data_ptr<scalar_t>(),
95 - key.data_ptr<scalar_t>(),
96 - cos_sin_cache.data_ptr<scalar_t>(),
97 - rot_dim,
98 - query_stride,
99 - key_stride,
100 - num_heads,
101 - num_kv_heads,
102 - head_size);
101 + if (is_neox) {
102 + vllm::rotary_embedding_kernel<scalar_t, true><<<grid, block, 0, stream>>>(
103 + positions.data_ptr<int64_t>(),
104 + query.data_ptr<scalar_t>(),
105 + key.data_ptr<scalar_t>(),
106 + cos_sin_cache.data_ptr<scalar_t>(),
107 + rot_dim,
108 + query_stride,
109 + key_stride,
110 + num_heads,
111 + num_kv_heads,
112 + head_size);
113 + } else {
114 + vllm::rotary_embedding_kernel<scalar_t, false><<<grid, block, 0, stream>>>(
115 + positions.data_ptr<int64_t>(),
116 + query.data_ptr<scalar_t>(),
117 + key.data_ptr<scalar_t>(),
118 + cos_sin_cache.data_ptr<scalar_t>(),
119 + rot_dim,
120 + query_stride,
121 + key_stride,
122 + num_heads,
123 + num_kv_heads,
124 + head_size);
125 + }
126 });
127 }
tests/kernels/test_pos_encoding.py
+38 -18
@@ -7,49 +7,64 @@ import torch.nn.functional as F
7
8 from vllm import pos_encoding_ops
9
10 +IS_NEOX_STYLE = [True, False]
11 DTYPES = [torch.half, torch.bfloat16, torch.float]
12 HEAD_SIZES = [64, 80, 96, 112, 128, 256]
13 ROTARY_DIMS = [None, 32] # None means rotary dim == head size
14 NUM_HEADS = [7, 12, 40, 52] # Arbitrary values for testing
14 -NUM_TOKENS = [7, 83, 2048] # Arbitrary values for testing
15 +NUM_TOKENS = [11, 83, 2048] # Arbitrary values for testing
16 SEEDS = [0]
17
18
18 -def rotate_half(x: torch.Tensor) -> torch.Tensor:
19 +def rotate_neox(x: torch.Tensor) -> torch.Tensor:
20 x1 = x[..., :x.shape[-1] // 2]
21 x2 = x[..., x.shape[-1] // 2:]
22 return torch.cat((-x2, x1), dim=-1)
23
24
24 -def apply_rotary_pos_emb(
25 +def rotate_gptj(x: torch.Tensor) -> torch.Tensor:
26 + x1 = x[..., ::2]
27 + x2 = x[..., 1::2]
28 + x = torch.stack((-x2, x1), dim=-1)
29 + return x.flatten(-2)
30 +
31 +
32 +def apply_rope(
33 q: torch.Tensor,
34 k: torch.Tensor,
35 cos: torch.Tensor,
36 sin: torch.Tensor,
37 + is_neox_style: bool,
38 ) -> Tuple[torch.Tensor, torch.Tensor]:
30 - q_embed = (q * cos) + (rotate_half(q) * sin)
31 - k_embed = (k * cos) + (rotate_half(k) * sin)
39 + rotate_fn = rotate_neox if is_neox_style else rotate_gptj
40 + q_embed = (q * cos) + (rotate_fn(q) * sin)
41 + k_embed = (k * cos) + (rotate_fn(k) * sin)
42 return q_embed, k_embed
43
44
35 -class RefRotaryEmbeddingNeox(nn.Module):
36 - """Reference implementation of the GPT-NeoX style rotary embedding."""
45 +class RefRotaryEmbedding(nn.Module):
46 + """Reference implementation of rotary embedding."""
47
48 def __init__(
49 self,
50 dim: int,
41 - max_position_embeddings: int = 2048,
51 + is_neox_style: bool,
52 + max_position_embeddings: int = 8192,
53 base: int = 10000,
54 ) -> None:
55 super().__init__()
56 self.rotary_dim = dim
57 + self.is_neox_style = is_neox_style
58 self.max_position_embeddings = max_position_embeddings
59
60 # Create cos and sin embeddings.
61 inv_freq = 1.0 / (base**(torch.arange(0, dim, 2) / dim))
62 t = torch.arange(max_position_embeddings).float()
63 freqs = torch.einsum("i,j->ij", t, inv_freq.float())
52 - emb = torch.cat((freqs, freqs), dim=-1)
64 + if is_neox_style:
65 + emb = torch.cat((freqs, freqs), dim=-1)
66 + else:
67 + emb = torch.repeat_interleave(freqs, 2, -1)
68 cos = emb.cos().to(dtype=inv_freq.dtype)
69 sin = emb.sin().to(dtype=inv_freq.dtype)
70 self.register_buffer("cos_cached", cos, persistent=False)
@@ -61,7 +76,6 @@ class RefRotaryEmbeddingNeox(nn.Module):
76 query: torch.Tensor, # [num_tokens, num_heads, head_size]
77 key: torch.Tensor, # [num_tokens, num_heads, head_size]
78 ) -> Tuple[torch.Tensor, torch.Tensor]:
64 -
79 query_rot = query[..., :self.rotary_dim]
80 query_pass = query[..., self.rotary_dim:]
81 key_rot = key[..., :self.rotary_dim]
@@ -71,7 +85,9 @@ class RefRotaryEmbeddingNeox(nn.Module):
85 key_rot = key_rot.transpose(0, 1)
86 cos = F.embedding(positions, self.cos_cached)
87 sin = F.embedding(positions, self.sin_cached)
74 - query_rot, key_rot = apply_rotary_pos_emb(query_rot, key_rot, cos, sin)
88 +
89 + query_rot, key_rot = apply_rope(query_rot, key_rot, cos, sin,
90 + self.is_neox_style)
91 query_rot = query_rot.transpose(0, 1).contiguous()
92 key_rot = key_rot.transpose(0, 1).contiguous()
93
@@ -82,6 +98,7 @@ class RefRotaryEmbeddingNeox(nn.Module):
98 return query, key
99
100
101 +@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
102 @pytest.mark.parametrize("num_tokens", NUM_TOKENS)
103 @pytest.mark.parametrize("num_heads", NUM_HEADS)
104 @pytest.mark.parametrize("head_size", HEAD_SIZES)
@@ -89,7 +106,8 @@ class RefRotaryEmbeddingNeox(nn.Module):
106 @pytest.mark.parametrize("dtype", DTYPES)
107 @pytest.mark.parametrize("seed", SEEDS)
108 @torch.inference_mode()
92 -def test_rotary_embedding_neox(
109 +def test_rotary_embedding(
110 + is_neox_style: bool,
111 num_tokens: int,
112 num_heads: int,
113 head_size: int,
@@ -104,15 +122,15 @@ def test_rotary_embedding_neox(
122 torch.random.manual_seed(seed)
123 torch.cuda.manual_seed(seed)
124
107 - positions = torch.randint(0, max_position, (num_tokens, ), device='cuda')
125 + positions = torch.randint(0, max_position, (num_tokens, ), device="cuda")
126 query = torch.randn(num_tokens,
127 num_heads * head_size,
128 dtype=dtype,
111 - device='cuda')
129 + device="cuda")
130 key = torch.randn(num_tokens,
131 num_heads * head_size,
132 dtype=dtype,
115 - device='cuda')
133 + device="cuda")
134
135 # Create the rotary embedding.
136 inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
@@ -126,20 +144,22 @@ def test_rotary_embedding_neox(
144 # Run the kernel. The kernel is in-place, so we need to clone the inputs.
145 out_query = query.clone()
146 out_key = key.clone()
129 - pos_encoding_ops.rotary_embedding_neox(
147 + pos_encoding_ops.rotary_embedding(
148 positions,
149 out_query,
150 out_key,
151 head_size,
152 cos_sin_cache,
153 + is_neox_style,
154 )
155
156 # Run the reference implementation.
138 - ref_rotary_embedding = RefRotaryEmbeddingNeox(
157 + ref_rotary_embedding = RefRotaryEmbedding(
158 dim=rotary_dim,
159 + is_neox_style=is_neox_style,
160 max_position_embeddings=max_position,
161 base=base,
142 - ).to(dtype=dtype, device='cuda')
162 + ).to(dtype=dtype, device="cuda")
163 ref_query, ref_key = ref_rotary_embedding(
164 positions,
165 query.view(num_tokens, num_heads, head_size),
vllm/model_executor/layers/attention.py
+5 -2
@@ -242,7 +242,7 @@ class PagedAttention(nn.Module):
242
243
244 class PagedAttentionWithRoPE(PagedAttention):
245 - """PagedAttention with GPT-NeoX style rotary embedding."""
245 + """PagedAttention with rotary embedding."""
246
247 def __init__(
248 self,
@@ -253,8 +253,10 @@ class PagedAttentionWithRoPE(PagedAttention):
253 max_position: int = 8192,
254 base: int = 10000,
255 num_kv_heads: Optional[int] = None,
256 + is_neox_style: bool = True,
257 ) -> None:
258 super().__init__(num_heads, head_size, scale, num_kv_heads)
259 + self.is_neox_style = is_neox_style
260
261 # Create the cos and sin cache.
262 inv_freq = 1.0 / (base**(torch.arange(0, rotary_dim, 2) / rotary_dim))
@@ -303,12 +305,13 @@ class PagedAttentionWithRoPE(PagedAttention):
305
306 # Apply rotary embedding to the query and key before passing them
307 # to the attention op.
306 - pos_encoding_ops.rotary_embedding_neox(
308 + pos_encoding_ops.rotary_embedding(
309 positions,
310 query,
311 key,
312 self.head_size,
313 self.cos_sin_cache,
314 + self.is_neox_style,
315 )
316 return super().forward(
317 query,
vllm/model_executor/models/gpt_j.py
+5 -2
@@ -67,8 +67,11 @@ class GPTJAttention(nn.Module):
67 scaling = self.head_size**-0.5
68 assert getattr(config, "rotary", True)
69 assert config.rotary_dim % 2 == 0
70 - self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_size,
71 - scaling, config.rotary_dim)
70 + self.attn = PagedAttentionWithRoPE(self.num_heads,
71 + self.head_size,
72 + scaling,
73 + config.rotary_dim,
74 + is_neox_style=False)
75 self.warmup = False
76
77 def forward(