A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Implement custom kernel for LLaMA rotary embedding (#14)
Woosuk Kwon committed
Mar 30, 2023 at 11:04 UTC
88c0268a18f1c724d59a624364635d5c7ac39408
10 files changed
+318
-69
cacheflow/models/attention.py
+71
-2
@@ -6,13 +6,14 @@ import torch.nn as nn
6
7
from cacheflow import attention_ops
8
from cacheflow import cache_ops
9
+from cacheflow import pos_encoding_ops
10
from cacheflow.models import InputMetadata
11
12
12
-class OPTCacheFlowAttention(nn.Module):
13
+class GPTCacheFlowAttention(nn.Module):
14
15
def __init__(self, scale: float) -> None:
15
- super(OPTCacheFlowAttention, self).__init__()
16
+ super().__init__()
17
self.scale = float(scale)
18
19
self.flash_attn = FlashAttention(softmax_scale=self.scale)
@@ -136,3 +137,71 @@ class OPTCacheFlowAttention(nn.Module):
137
# Reshape the output tensor.
138
# NOTE(woosuk): The output tensor may include paddings.
139
return output.view(-1, num_heads * head_size)
140
+
141
+
142
+class OPTCacheFlowAttention(GPTCacheFlowAttention):
143
+ """OPT uses the same attention mechanism as GPT."""
144
+
145
+ def __init__(self, scale: float) -> None:
146
+ super().__init__(scale)
147
+
148
+
149
+class LlamaCacheFlowAttention(GPTCacheFlowAttention):
150
+ """Llama uses GPT-NeoX style rotary embedding."""
151
+
152
+ def __init__(
153
+ self,
154
+ scale: float,
155
+ head_size: int,
156
+ max_position: int = 8192,
157
+ base: int = 10000,
158
+ ) -> None:
159
+ super().__init__(scale)
160
+
161
+ # Create the cos and sin cache.
162
+ inv_freq = 1.0 / (base ** (torch.arange(0, head_size, 2) / head_size))
163
+ t = torch.arange(max_position).float()
164
+ freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
165
+ cos = freqs.cos()
166
+ sin = freqs.sin()
167
+ cache = torch.cat((cos, sin), dim=-1)
168
+
169
+ # FIXME(woosuk): This assumes that we configure the default dtype when
170
+ # initializing the model. Make it more robust.
171
+ torch_dtype = torch.get_default_dtype()
172
+ cache = cache.to(torch_dtype)
173
+ # Embedding size: [max_position, head_size]
174
+ self.register_buffer('cos_sin_cache', cache, persistent=False)
175
+
176
+ def forward(
177
+ self,
178
+ positions: torch.LongTensor, # [num_tokens]
179
+ query: torch.Tensor, # [num_tokens, num_heads * head_size]
180
+ key: torch.Tensor, # [num_tokens, num_heads * head_size]
181
+ value: torch.Tensor, # [num_tokens, num_heads * head_size]
182
+ key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
183
+ value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
184
+ input_metadata: InputMetadata,
185
+ cache_event: Optional[torch.cuda.Event],
186
+ ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
187
+ # Apply rotary embedding to the query and key before passing them
188
+ # to the attention op.
189
+ out_query = torch.empty_like(query)
190
+ out_key = torch.empty_like(key)
191
+ pos_encoding_ops.rotary_embedding_neox(
192
+ out_query,
193
+ out_key,
194
+ positions,
195
+ query,
196
+ key,
197
+ self.cos_sin_cache,
198
+ )
199
+ return super().forward(
200
+ out_query,
201
+ out_key,
202
+ value,
203
+ key_cache,
204
+ value_cache,
205
+ input_metadata,
206
+ cache_event,
207
+ )
cacheflow/models/llama.py
+5
-59
@@ -8,12 +8,10 @@ from typing import Dict, List, Optional, Tuple
8
import numpy as np
9
import torch
10
from torch import nn
11
-import torch.nn.functional as F
11
from transformers import LlamaConfig
13
-from transformers import PreTrainedModel
12
13
from cacheflow.models import InputMetadata
16
-from cacheflow.models.attention import OPTCacheFlowAttention
14
+from cacheflow.models.attention import LlamaCacheFlowAttention
15
from cacheflow.models.sample import Sampler
16
from cacheflow.parallel_utils.parallel_state import (
17
get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
@@ -41,48 +39,8 @@ class LlamaRMSNorm(nn.Module):
39
return self.weight * hidden_states
40
41
44
-class LlamaRotaryEmbedding(torch.nn.Module):
45
-
46
- def __init__(self, dim, max_position_embeddings=2048, base=10000):
47
- super().__init__()
48
- self.max_position_embeddings = max_position_embeddings
49
-
50
- inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2) / dim))
51
- self.register_buffer("inv_freq", inv_freq)
52
-
53
- # Create cos and sin embeddings.
54
- t = torch.arange(max_position_embeddings).float()
55
- freqs = torch.einsum("i,j->ij", t, self.inv_freq.float())
56
- emb = torch.cat((freqs, freqs), dim=-1)
57
- cos = emb.cos().to(dtype=self.inv_freq.dtype)
58
- sin = emb.sin().to(dtype=self.inv_freq.dtype)
59
- self.register_buffer("cos_cached", cos, persistent=False)
60
- self.register_buffer("sin_cached", sin, persistent=False)
61
-
62
- def forward(
63
- self,
64
- positions: torch.LongTensor,
65
- ) -> Tuple[torch.Tensor, torch.Tensor]:
66
- cos = F.embedding(positions, self.cos_cached)
67
- sin = F.embedding(positions, self.sin_cached)
68
- return cos, sin
69
-
70
-
71
-def rotate_half(x):
72
- """Rotates half the hidden dims of the input."""
73
- x1 = x[..., : x.shape[-1] // 2]
74
- x2 = x[..., x.shape[-1] // 2 :]
75
- return torch.cat((-x2, x1), dim=-1)
76
-
77
-
78
-def apply_rotary_pos_emb(q, k, cos, sin):
79
- # TODO: Optimize.
80
- q_embed = (q * cos) + (rotate_half(q) * sin)
81
- k_embed = (k * cos) + (rotate_half(k) * sin)
82
- return q_embed, k_embed
83
-
84
-
42
class LlamaMLP(nn.Module):
43
+
44
def __init__(
45
self,
46
hidden_size: int,
@@ -156,9 +114,7 @@ class LlamaAttention(nn.Module):
114
input_is_parallel=True,
115
perform_initialization=False,
116
)
159
- self.rotary_emb = LlamaRotaryEmbedding(self.head_dim)
160
- # FIXME(woosuk): Rename this.
161
- self.attn = OPTCacheFlowAttention(scale=self.scaling)
117
+ self.attn = LlamaCacheFlowAttention(self.scaling, self.head_dim)
118
119
def forward(
120
self,
@@ -171,19 +127,9 @@ class LlamaAttention(nn.Module):
127
q, _ = self.q_proj(hidden_states)
128
k, _ = self.k_proj(hidden_states)
129
v, _ = self.v_proj(hidden_states)
174
-
175
- # Apply rotrary embedding.
176
- # TODO: Optimize.
177
- q = q.view(-1, self.num_heads, self.head_dim).transpose(0, 1)
178
- k = k.view(-1, self.num_heads, self.head_dim).transpose(0, 1)
179
- cos, sin = self.rotary_emb(positions)
180
- q, k = apply_rotary_pos_emb(q, k, cos, sin)
181
- q = q.transpose(0, 1).contiguous().view(-1, self.num_heads * self.head_dim)
182
- k = k.transpose(0, 1).contiguous().view(-1, self.num_heads * self.head_dim)
183
-
184
- key_cache, value_cache = kv_cache
130
+ k_cache, v_cache = kv_cache
131
attn_output = self.attn(
186
- q, k, v, key_cache, value_cache, input_metadata, cache_event)
132
+ positions, q, k, v, k_cache, v_cache, input_metadata, cache_event)
133
output, _ = self.o_proj(attn_output)
134
return output
135
cacheflow/models/memory_analyzer.py
+1
-2
@@ -165,8 +165,7 @@ class LlamaMemoryAnalyzer(CacheFlowMemoryAnalyzer):
165
self.head_size = config.hidden_size // self.num_heads
166
self.ffn_size = config.intermediate_size
167
self.vocab_size = config.vocab_size
168
- # FIXME
169
- self.max_position = 2048
168
+ self.max_position = 8192
169
170
def _get_param_size(self) -> int:
171
word_embedding = self.vocab_size * self.hidden_size // self.tensor_parallel_size
cacheflow/models/opt.py
+1
-2
@@ -51,7 +51,7 @@ class OPTAttention(nn.Module):
51
assert num_heads % tensor_model_parallel_world_size == 0
52
self.num_heads = total_num_heads // tensor_model_parallel_world_size
53
self.head_dim = embed_dim // total_num_heads
54
- self.scaling = self.head_dim**-0.5
54
+ self.scaling = self.head_dim ** -0.5
55
56
# TODO(woosuk): Fuse the three linear layers into one QKV linear layer.
57
self.k_proj = ColumnParallelLinear(embed_dim, embed_dim, bias=bias,
@@ -66,7 +66,6 @@ class OPTAttention(nn.Module):
66
self.out_proj = RowParallelLinear(embed_dim, embed_dim, bias=bias,
67
input_is_parallel=True,
68
perform_initialization=False)
69
-
69
self.attn = OPTCacheFlowAttention(scale=self.scaling)
70
71
def forward(
cacheflow/models/sample.py
+1
-1
@@ -12,7 +12,7 @@ from cacheflow.parallel_utils.tensor_parallel import gather_from_tensor_model_pa
12
class Sampler(nn.Module):
13
14
def __init__(self) -> None:
15
- super(Sampler, self).__init__()
15
+ super().__init__()
16
17
def forward(
18
self,
csrc/cache_kernels.cu
+3
-3
@@ -122,13 +122,13 @@ void reshape_and_cache(
122
torch::Tensor& value_cache,
123
torch::Tensor& slot_mapping) {
124
int num_tokens = key.size(0);
125
- int head_num = key.size(1);
125
+ int num_heads = key.size(1);
126
int head_size = key.size(2);
127
int block_size = key_cache.size(3);
128
int x = key_cache.size(4);
129
130
dim3 grid(num_tokens);
131
- dim3 block(std::min(head_num * head_size, 512));
131
+ dim3 block(std::min(num_heads * head_size, 512));
132
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
133
AT_DISPATCH_FLOATING_TYPES_AND_HALF(
134
key.scalar_type(),
@@ -140,7 +140,7 @@ void reshape_and_cache(
140
key_cache.data_ptr<scalar_t>(),
141
value_cache.data_ptr<scalar_t>(),
142
slot_mapping.data_ptr<int>(),
143
- head_num,
143
+ num_heads,
144
head_size,
145
block_size,
146
x);
csrc/pos_encoding.cpp
new
+16
@@ -0,0 +1,16 @@
1
+#include <torch/extension.h>
2
+
3
+void rotary_embedding_neox(
4
+ torch::Tensor& out_query,
5
+ torch::Tensor& out_key,
6
+ torch::Tensor& positions,
7
+ torch::Tensor& query,
8
+ torch::Tensor& key,
9
+ torch::Tensor& cos_sin_cache);
10
+
11
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
12
+ m.def(
13
+ "rotary_embedding_neox",
14
+ &rotary_embedding_neox,
15
+ "Apply GPT-NeoX style rotary embedding to query and key");
16
+}
csrc/pos_encoding_kernels.cu
new
+83
@@ -0,0 +1,83 @@
1
+#include <torch/extension.h>
2
+#include <ATen/cuda/CUDAContext.h>
3
+
4
+namespace cacheflow {
5
+
6
+template<typename scalar_t>
7
+__global__ void rotary_embedding_neox_kernel(
8
+ scalar_t* __restrict__ out_query, // [num_tokens, num_heads, head_size]
9
+ scalar_t* __restrict__ out_key, // [num_tokens, num_heads, head_size]
10
+ const int64_t* __restrict__ positions, // [num_tokens]
11
+ const scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
12
+ const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
13
+ const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, head_size // 2]
14
+ const int num_heads,
15
+ const int head_size) {
16
+ // Each thread block is responsible for one token.
17
+ const int token_idx = blockIdx.x;
18
+ int64_t pos = positions[token_idx];
19
+ const scalar_t* cache_ptr = cos_sin_cache + pos * head_size;
20
+
21
+ const int embed_dim = head_size / 2;
22
+ const int n = num_heads * head_size;
23
+ for (int i = threadIdx.x; i < n; i += blockDim.x) {
24
+ const int idx = token_idx * n + i;
25
+
26
+ const int head_idx = i / head_size;
27
+ const int head_offset = i % head_size;
28
+ const int token_head = token_idx * n + head_idx * head_size;
29
+
30
+ const bool is_first_half = head_offset < embed_dim;
31
+ const int rot_offset = head_offset % embed_dim;
32
+ const int x_index = rot_offset;
33
+ const int y_index = embed_dim + rot_offset;
34
+
35
+ const scalar_t cos = __ldg(cache_ptr + x_index);
36
+ const scalar_t sin = __ldg(cache_ptr + y_index);
37
+
38
+ const scalar_t q_x = __ldg(query + token_head + x_index);
39
+ const scalar_t q_y = __ldg(query + token_head + y_index);
40
+ const scalar_t q_cos = is_first_half ? q_x : q_y;
41
+ const scalar_t q_sin = is_first_half ? -q_y : q_x;
42
+ out_query[idx] = q_cos * cos + q_sin * sin;
43
+
44
+ const scalar_t k_x = __ldg(key + token_head + x_index);
45
+ const scalar_t k_y = __ldg(key + token_head + y_index);
46
+ const scalar_t k_cos = is_first_half ? k_x : k_y;
47
+ const scalar_t k_sin = is_first_half ? -k_y : k_x;
48
+ out_key[idx] = k_cos * cos + k_sin * sin;
49
+ }
50
+}
51
+
52
+} // namespace cacheflow
53
+
54
+void rotary_embedding_neox(
55
+ torch::Tensor& out_query, // [num_tokens, num_heads * head_size]
56
+ torch::Tensor& out_key, // [num_tokens, num_heads * head_size]
57
+ torch::Tensor& positions, // [num_tokens]
58
+ torch::Tensor& query, // [num_tokens, num_heads * head_size]
59
+ torch::Tensor& key, // [num_tokens, num_heads * head_size]
60
+ torch::Tensor& cos_sin_cache) // [max_position, head_size]
61
+{
62
+ int num_tokens = query.size(0);
63
+ int head_size = cos_sin_cache.size(1);
64
+ int num_heads = query.size(1) / head_size;
65
+
66
+ dim3 grid(num_tokens);
67
+ dim3 block(std::min(num_heads * head_size, 512));
68
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
69
+ AT_DISPATCH_FLOATING_TYPES_AND_HALF(
70
+ query.scalar_type(),
71
+ "rotary_embedding_neox",
72
+ [&] {
73
+ cacheflow::rotary_embedding_neox_kernel<scalar_t><<<grid, block, 0, stream>>>(
74
+ out_query.data_ptr<scalar_t>(),
75
+ out_key.data_ptr<scalar_t>(),
76
+ positions.data_ptr<int64_t>(),
77
+ query.data_ptr<scalar_t>(),
78
+ key.data_ptr<scalar_t>(),
79
+ cos_sin_cache.data_ptr<scalar_t>(),
80
+ num_heads,
81
+ head_size);
82
+ });
83
+}
setup.py
+8
@@ -23,6 +23,14 @@ attention_extension = cpp_extension.CUDAExtension(
23
)
24
ext_modules.append(attention_extension)
25
26
+# Positional encodings.
27
+positional_encoding_extension = cpp_extension.CUDAExtension(
28
+ name='cacheflow.pos_encoding_ops',
29
+ sources=['csrc/pos_encoding.cpp', 'csrc/pos_encoding_kernels.cu'],
30
+ extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
31
+)
32
+ext_modules.append(positional_encoding_extension)
33
+
34
setuptools.setup(
35
name='cacheflow',
36
ext_modules=ext_modules,
tests/kernels/pos_encoding.py
new
+129
@@ -0,0 +1,129 @@
1
+from typing import Tuple
2
+
3
+import torch
4
+import torch.nn as nn
5
+import torch.nn.functional as F
6
+
7
+from cacheflow import pos_encoding_ops
8
+
9
+
10
+def rotate_half(x: torch.Tensor) -> torch.Tensor:
11
+ x1 = x[..., : x.shape[-1] // 2]
12
+ x2 = x[..., x.shape[-1] // 2 :]
13
+ return torch.cat((-x2, x1), dim=-1)
14
+
15
+
16
+def apply_rotary_pos_emb(
17
+ q: torch.Tensor,
18
+ k: torch.Tensor,
19
+ cos: torch.Tensor,
20
+ sin: torch.Tensor,
21
+) -> Tuple[torch.Tensor, torch.Tensor]:
22
+ q_embed = (q * cos) + (rotate_half(q) * sin)
23
+ k_embed = (k * cos) + (rotate_half(k) * sin)
24
+ return q_embed, k_embed
25
+
26
+
27
+class RefRotaryEmbeddingNeox(nn.Module):
28
+ """Reference implementation of the GPT-NeoX style rotary embedding."""
29
+
30
+ def __init__(
31
+ self,
32
+ dim: int,
33
+ max_position_embeddings: int = 2048,
34
+ base: int = 10000,
35
+ ) -> None:
36
+ super().__init__()
37
+ self.max_position_embeddings = max_position_embeddings
38
+
39
+ # Create cos and sin embeddings.
40
+ inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2) / dim))
41
+ t = torch.arange(max_position_embeddings).float()
42
+ freqs = torch.einsum("i,j->ij", t, inv_freq.float())
43
+ emb = torch.cat((freqs, freqs), dim=-1)
44
+ cos = emb.cos().to(dtype=inv_freq.dtype)
45
+ sin = emb.sin().to(dtype=inv_freq.dtype)
46
+ self.register_buffer("cos_cached", cos, persistent=False)
47
+ self.register_buffer("sin_cached", sin, persistent=False)
48
+
49
+ def forward(
50
+ self,
51
+ positions: torch.LongTensor, # [num_tokens]
52
+ query: torch.Tensor, # [num_tokens, num_heads, head_size]
53
+ key: torch.Tensor, # [num_tokens, num_heads, head_size]
54
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
55
+ cos = F.embedding(positions, self.cos_cached)
56
+ sin = F.embedding(positions, self.sin_cached)
57
+ query = query.transpose(0, 1)
58
+ key = key.transpose(0, 1)
59
+ query, key = apply_rotary_pos_emb(query, key, cos, sin)
60
+ query = query.transpose(0, 1).contiguous()
61
+ key = key.transpose(0, 1).contiguous()
62
+ # Output query/key shape: [num_tokens, num_tokens, head_size]
63
+ return query, key
64
+
65
+
66
+@torch.inference_mode()
67
+def test_rotary_embedding_neox(
68
+ num_tokens: int,
69
+ num_heads: int,
70
+ head_size: int,
71
+ max_position: int,
72
+ dtype: torch.dtype,
73
+ base: int = 10000,
74
+) -> None:
75
+ positions = torch.randint(0, max_position, (num_tokens,), device='cuda')
76
+ query = torch.randn(num_tokens, num_heads * head_size, dtype=dtype, device='cuda')
77
+ key = torch.randn(num_tokens, num_heads * head_size, dtype=dtype, device='cuda')
78
+
79
+ # Create the rotary embedding.
80
+ inv_freq = 1.0 / (base ** (torch.arange(0, head_size, 2) / head_size))
81
+ t = torch.arange(max_position).float()
82
+ freqs = torch.einsum('i,j -> ij', t, inv_freq.float())
83
+ cos = freqs.cos()
84
+ sin = freqs.sin()
85
+ cos_sin_cache = torch.cat((cos, sin), dim=-1)
86
+ cos_sin_cache = cos_sin_cache.to(dtype=dtype, device='cuda')
87
+
88
+ # Run the kernel.
89
+ out_query = torch.empty_like(query)
90
+ out_key = torch.empty_like(key)
91
+ pos_encoding_ops.rotary_embedding_neox(
92
+ out_query,
93
+ out_key,
94
+ positions,
95
+ query,
96
+ key,
97
+ cos_sin_cache,
98
+ )
99
+
100
+ # Run the reference implementation.
101
+ ref_rotary_embedding = RefRotaryEmbeddingNeox(
102
+ dim=head_size,
103
+ max_position_embeddings=max_position,
104
+ base=base,
105
+ ).to(dtype=dtype, device='cuda')
106
+ ref_query, ref_key = ref_rotary_embedding(
107
+ positions,
108
+ query.view(num_tokens, num_heads, head_size),
109
+ key.view(num_tokens, num_heads, head_size),
110
+ )
111
+ ref_query = ref_query.view(num_tokens, num_heads * head_size)
112
+ ref_key = ref_key.view(num_tokens, num_heads * head_size)
113
+
114
+ # Compare the results.
115
+ assert torch.allclose(out_query, ref_query, atol=1e-3, rtol=1e-5)
116
+ assert torch.allclose(out_key, ref_key, atol=1e-3, rtol=1e-5)
117
+
118
+
119
+if __name__ == '__main__':
120
+ for dtype in [torch.half, torch.float]:
121
+ for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
122
+ print(f'Running tests for head_size={head_size} and dtype={dtype}')
123
+ test_rotary_embedding_neox(
124
+ num_tokens=2145,
125
+ num_heads=5,
126
+ head_size=head_size,
127
+ max_position=8192,
128
+ dtype=dtype,
129
+ )