A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Optimize data movement (#20)
Woosuk Kwon committed
Apr 2, 2023 at 00:30 UTC
897cb2ae28e93de1b22ecfbffcccfb9493f8f4d9
17 files changed
+275
-135
cacheflow/models/activation.py
new
+20
@@ -0,0 +1,20 @@
1
+import torch
2
+import torch.nn as nn
3
+
4
+from cacheflow import activation_ops
5
+
6
+
7
+class SiluAndMul(nn.Module):
8
+
9
+ def __init__(self):
10
+ super().__init__()
11
+
12
+ def forward(
13
+ self,
14
+ x: torch.Tensor, # (num_tokens, 2 * d)
15
+ ) -> torch.Tensor: # (num_tokens, d)
16
+ num_tokens = x.shape[0]
17
+ d = x.shape[1] // 2
18
+ out = torch.empty(num_tokens, d, dtype=x.dtype, device=x.device)
19
+ activation_ops.silu_and_mul(out, x)
20
+ return out
cacheflow/models/attention.py
+46
-46
@@ -1,6 +1,6 @@
1
-from typing import List, Optional
1
+from typing import Optional
2
3
-from flash_attn.flash_attention import FlashAttention
3
+from flash_attn.flash_attn_interface import _flash_attn_forward
4
import torch
5
import torch.nn as nn
6
@@ -16,40 +16,38 @@ class GPTCacheFlowAttention(nn.Module):
16
super().__init__()
17
self.scale = float(scale)
18
19
- self.flash_attn = FlashAttention(softmax_scale=self.scale)
20
-
19
def multi_query_kv_attention(
20
self,
23
- output: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
24
- query: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
25
- key: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
26
- value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
27
- prompt_lens: List[int],
21
+ output: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
22
+ query: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
23
+ key: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
24
+ value: torch.Tensor, # [num_prompt_tokens, num_heads, head_size]
25
+ cumulative_prompt_lens: torch.Tensor, # [num_prompts + 1]
26
+ max_prompt_len: int,
27
) -> None:
28
if query.dtype == torch.float:
29
raise ValueError('The float data type is not supported by '
30
'FlashAttention. Use the half data type instead.')
32
- head_size = query.shape[2]
31
+ head_size = query.shape[-1]
32
if head_size > 128:
33
raise ValueError('FlashAttention does not support head_size > 128.')
34
36
- device = query.device
37
- prefix_sum = [0]
38
- for prompt_len in prompt_lens:
39
- prefix_sum.append(prefix_sum[-1] + prompt_len)
40
- prefix_sum = torch.tensor(prefix_sum, dtype=torch.int, device=device)
41
- max_prompt_len = max(prompt_lens)
42
-
43
- # FIXME(woosuk): Unnecessary copy. Optimize this.
44
- qkv = torch.stack([query, key, value], dim=1)
45
- out = self.flash_attn(
46
- qkv,
47
- cu_seqlens=prefix_sum,
48
- max_s=max_prompt_len,
35
+ # Directly call FlashAttention's internal function to avoid allocating
36
+ # a new tensor for the output.
37
+ _flash_attn_forward(
38
+ query,
39
+ key,
40
+ value,
41
+ output,
42
+ cumulative_prompt_lens,
43
+ cumulative_prompt_lens,
44
+ max_prompt_len,
45
+ max_prompt_len,
46
+ dropout_p=0.0,
47
+ softmax_scale=self.scale,
48
causal=True,
50
- )[0]
51
- # FIXME(woosuk): Unnecessary copy. Optimize this.
52
- output.copy_(out, non_blocking=True)
49
+ return_softmax=False,
50
+ )
51
52
def single_query_cached_kv_attention(
53
self,
@@ -90,21 +88,18 @@ class GPTCacheFlowAttention(nn.Module):
88
input_metadata: InputMetadata,
89
cache_event: Optional[torch.cuda.Event],
90
) -> torch.Tensor: # [num_tokens, num_heads * head_size]
93
- # Pre-allocate the output tensor.
94
- output = torch.empty_like(query)
95
-
96
- # Prune out paddings if any.
97
- query = query[:input_metadata.num_valid_tokens]
98
- key = key[:input_metadata.num_valid_tokens]
99
- value = value[:input_metadata.num_valid_tokens]
91
+ # NOTE: The query, key, and value tensors must be sliced from a qkv
92
+ # tensor of shape [num_tokens, 3 * num_heads * head_size].
93
101
- # Reshape the input tensors.
94
+ # Reshape the query, key, and value tensors.
95
num_heads = value_cache.shape[1]
96
head_size = value_cache.shape[2]
97
query = query.view(-1, num_heads, head_size)
98
key = key.view(-1, num_heads, head_size)
99
value = value.view(-1, num_heads, head_size)
107
- output = output.view(-1, num_heads, head_size)
100
+
101
+ # Pre-allocate the output tensor.
102
+ output = torch.empty_like(query)
103
104
# Compute the attention op for prompts.
105
num_prompt_tokens = input_metadata.num_prompt_tokens
@@ -114,7 +109,8 @@ class GPTCacheFlowAttention(nn.Module):
109
query[:num_prompt_tokens],
110
key[:num_prompt_tokens],
111
value[:num_prompt_tokens],
117
- input_metadata.prompt_lens,
112
+ input_metadata.cumulative_prompt_lens,
113
+ input_metadata.max_prompt_len,
114
)
115
116
# Wait until the cache op is done.
@@ -122,14 +118,22 @@ class GPTCacheFlowAttention(nn.Module):
118
cache_event.wait()
119
120
# Reshape the keys and values and store them in the cache.
125
- cache_ops.reshape_and_cache(
126
- key, value, key_cache, value_cache, input_metadata.slot_mapping)
121
+ num_valid_tokens = input_metadata.num_valid_tokens
122
+ if num_valid_tokens > 0:
123
+ # The stride is 3 because the key and value are sliced from qkv.
124
+ cache_ops.reshape_and_cache(
125
+ key[:num_valid_tokens],
126
+ value[:num_valid_tokens],
127
+ key_cache,
128
+ value_cache,
129
+ input_metadata.slot_mapping,
130
+ )
131
132
if input_metadata.num_generation_tokens > 0:
133
# Compute the attention op for generation tokens.
134
self.single_query_cached_kv_attention(
131
- output[num_prompt_tokens:],
132
- query[num_prompt_tokens:],
135
+ output[num_prompt_tokens:num_valid_tokens],
136
+ query[num_prompt_tokens:num_valid_tokens],
137
key_cache,
138
value_cache,
139
input_metadata)
@@ -186,19 +190,15 @@ class LlamaCacheFlowAttention(GPTCacheFlowAttention):
190
) -> torch.Tensor: # [num_tokens, num_heads * head_size]
191
# Apply rotary embedding to the query and key before passing them
192
# to the attention op.
189
- out_query = torch.empty_like(query)
190
- out_key = torch.empty_like(key)
193
pos_encoding_ops.rotary_embedding_neox(
192
- out_query,
193
- out_key,
194
positions,
195
query,
196
key,
197
self.cos_sin_cache,
198
)
199
return super().forward(
200
- out_query,
201
- out_key,
200
+ query,
201
+ key,
202
value,
203
key_cache,
204
value_cache,
cacheflow/models/input_metadata.py
+5
@@ -12,6 +12,7 @@ class InputMetadata:
12
seq_groups: List[Tuple[List[int], SamplingParams]],
13
seq_logprobs: Dict[int, float], # Seq id -> cumulative logprobs.
14
prompt_lens: List[int],
15
+ cumulative_prompt_lens: torch.Tensor,
16
slot_mapping: torch.Tensor,
17
context_lens: torch.Tensor,
18
max_context_len: int,
@@ -20,6 +21,7 @@ class InputMetadata:
21
self.seq_groups = seq_groups
22
self.seq_logprobs = seq_logprobs
23
self.prompt_lens = prompt_lens
24
+ self.cumulative_prompt_lens = cumulative_prompt_lens
25
self.slot_mapping = slot_mapping
26
self.context_lens = context_lens
27
self.max_context_len = max_context_len
@@ -27,6 +29,7 @@ class InputMetadata:
29
30
self.num_prompts = len(prompt_lens)
31
self.num_prompt_tokens = sum(prompt_lens)
32
+ self.max_prompt_len = max(prompt_lens) if prompt_lens else 0
33
self.num_generation_tokens = context_lens.shape[0]
34
self.num_valid_tokens = slot_mapping.shape[0]
35
if block_tables.numel() > 0:
@@ -40,11 +43,13 @@ class InputMetadata:
43
return (f'InputMetadata('
44
f'num_prompts={self.num_prompts}, '
45
f'num_prompt_tokens={self.num_prompt_tokens}, '
46
+ f'max_prompt_len={self.max_prompt_len}, '
47
f'num_generation_tokens={self.num_generation_tokens}, '
48
f'num_valid_tokens={self.num_valid_tokens}, '
49
f'max_num_blocks_per_seq={self.max_num_blocks_per_seq}, '
50
f'max_context_len={self.max_context_len}), '
51
f'prompt_lens={self.prompt_lens}, '
52
+ f'cumulative_prompt_lens={self.cumulative_prompt_lens}, '
53
f'slot_mapping={self.slot_mapping}, '
54
f'context_lens={self.context_lens}, '
55
f'block_tables={self.block_tables})')
cacheflow/models/llama.py
+7
-12
@@ -11,6 +11,7 @@ from torch import nn
11
from transformers import LlamaConfig
12
13
from cacheflow.models import InputMetadata
14
+from cacheflow.models.activation import SiluAndMul
15
from cacheflow.models.attention import LlamaCacheFlowAttention
16
from cacheflow.models.layernorm import RMSNorm
17
from cacheflow.models.sample import Sampler
@@ -39,16 +40,14 @@ class LlamaMLP(nn.Module):
40
self.down_proj = RowParallelLinear(intermediate_size, hidden_size,
41
bias=False, input_is_parallel=True,
42
perform_initialization=False)
42
- assert hidden_act == 'silu'
43
- self.act_fn = nn.SiLU()
43
+ if hidden_act != 'silu':
44
+ raise ValueError(f'Unsupported activation: {hidden_act}. '
45
+ 'Only silu is supported for now.')
46
+ self.act_fn = SiluAndMul()
47
48
def forward(self, x):
49
gate_up, _ = self.gate_up_proj(x)
47
- gate_up = gate_up.reshape(gate_up.shape[:-1] + (2, -1))
48
- gate, up = torch.split(gate_up, 1, dim=-2)
49
- gate = gate.squeeze(dim=-2).contiguous()
50
- up = up.squeeze(dim=-2).contiguous()
51
- x = self.act_fn(gate) * up
50
+ x = self.act_fn(gate_up)
51
x, _ = self.down_proj(x)
52
return x
53
@@ -94,11 +93,7 @@ class LlamaAttention(nn.Module):
93
cache_event: Optional[torch.cuda.Event],
94
) -> torch.Tensor:
95
qkv, _ = self.qkv_proj(hidden_states)
97
- qkv = qkv.reshape(qkv.shape[:-1] + (3, -1))
98
- q, k, v = torch.split(qkv, 1, dim=-2)
99
- q = q.squeeze(dim=-2).contiguous()
100
- k = k.squeeze(dim=-2).contiguous()
101
- v = v.squeeze(dim=-2).contiguous()
96
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
97
k_cache, v_cache = kv_cache
98
attn_output = self.attn(
99
positions, q, k, v, k_cache, v_cache, input_metadata, cache_event)
cacheflow/models/opt.py
+2
-5
@@ -69,17 +69,14 @@ class OPTAttention(nn.Module):
69
cache_event: Optional[torch.cuda.Event],
70
) -> torch.Tensor:
71
qkv, _ = self.qkv_proj(hidden_states)
72
- qkv = qkv.reshape(qkv.shape[:-1] + (3, -1))
73
- q, k, v = torch.split(qkv, 1, dim=-2)
74
- q = q.squeeze(dim=-2).contiguous()
75
- k = k.squeeze(dim=-2).contiguous()
76
- v = v.squeeze(dim=-2).contiguous()
72
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
73
key_cache, value_cache = kv_cache
74
attn_output = self.attn(
75
q, k, v, key_cache, value_cache, input_metadata, cache_event)
76
output, _ = self.out_proj(attn_output)
77
return output
78
79
+
80
class OPTDecoderLayer(nn.Module):
81
82
def __init__(self, config: OPTConfig):
cacheflow/worker/worker.py
+8
@@ -128,6 +128,11 @@ class Worker:
128
slot = block_number * self.block_size + block_offset
129
slot_mapping.append(slot)
130
131
+ cumulative_prompt_lens: List[int] = [0]
132
+ for prompt_len in prompt_lens:
133
+ cumulative_prompt_lens.append(
134
+ cumulative_prompt_lens[-1] + prompt_len)
135
+
136
# Add generation tokens.
137
max_context_len = 0
138
max_num_blocks_per_seq = 0
@@ -183,11 +188,14 @@ class Worker:
188
for block_table in generation_block_tables]
189
block_tables_tensor = torch.tensor(
190
padded_block_tables, dtype=torch.int, device='cuda')
191
+ cumulative_prompt_lens_tensor = torch.tensor(
192
+ cumulative_prompt_lens, dtype=torch.int, device='cuda')
193
194
input_metadata = InputMetadata(
195
seq_groups=seq_groups,
196
seq_logprobs=seq_logprobs,
197
prompt_lens=prompt_lens,
198
+ cumulative_prompt_lens=cumulative_prompt_lens_tensor,
199
slot_mapping=slot_mapping_tensor,
200
context_lens=context_lens_tensor,
201
max_context_len=max_context_len,
csrc/activation.cpp
new
+12
@@ -0,0 +1,12 @@
1
+#include <torch/extension.h>
2
+
3
+void silu_and_mul(
4
+ torch::Tensor& out,
5
+ torch::Tensor& input);
6
+
7
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
8
+ m.def(
9
+ "silu_and_mul",
10
+ &silu_and_mul,
11
+ "Activation function used in SwiGLU.");
12
+}
csrc/activation_kernels.cu
new
+46
@@ -0,0 +1,46 @@
1
+#include <torch/extension.h>
2
+#include <ATen/cuda/CUDAContext.h>
3
+
4
+namespace cacheflow {
5
+
6
+template<typename T>
7
+__device__ __forceinline__ T silu(const T& x) {
8
+ // x * sigmoid(x)
9
+ return (T) (((float) x) / (1.0f + expf((float) -x)));
10
+}
11
+
12
+template<typename scalar_t>
13
+__global__ void silu_and_mul_kernel(
14
+ scalar_t* __restrict__ out, // [num_tokens, d]
15
+ const scalar_t* __restrict__ input, // [num_tokens, 2, d]
16
+ const int d) {
17
+ const int token_idx = blockIdx.x;
18
+ for (int idx = threadIdx.x; idx < d; idx += blockDim.x) {
19
+ const scalar_t x = __ldg(&input[token_idx * 2 * d + idx]);
20
+ const scalar_t y = __ldg(&input[token_idx * 2 * d + d + idx]);
21
+ out[token_idx * d + idx] = silu(x) * y;
22
+ }
23
+}
24
+
25
+} // namespace cacheflow
26
+
27
+void silu_and_mul(
28
+ torch::Tensor& out, // [num_tokens, d]
29
+ torch::Tensor& input) // [num_tokens, 2 * d]
30
+{
31
+ int num_tokens = input.size(0);
32
+ int d = input.size(1) / 2;
33
+
34
+ dim3 grid(num_tokens);
35
+ dim3 block(std::min(d, 1024));
36
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
37
+ AT_DISPATCH_FLOATING_TYPES_AND_HALF(
38
+ input.scalar_type(),
39
+ "silu_and_mul_kernel",
40
+ [&] {
41
+ cacheflow::silu_and_mul_kernel<scalar_t><<<grid, block, 0, stream>>>(
42
+ out.data_ptr<scalar_t>(),
43
+ input.data_ptr<scalar_t>(),
44
+ d);
45
+ });
46
+}
csrc/attention_kernels.cu
+13
-9
@@ -25,7 +25,8 @@ __global__ void single_query_cached_kv_attention_kernel(
25
const float scale,
26
const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
27
const int* __restrict__ context_lens, // [num_seqs]
28
- const int max_num_blocks_per_seq) {
28
+ const int max_num_blocks_per_seq,
29
+ const int q_stride) {
30
constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
31
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
32
const int thread_idx = threadIdx.x;
@@ -56,7 +57,8 @@ __global__ void single_query_cached_kv_attention_kernel(
57
// For example, if the the thread group size is 4, then the first thread in the group
58
// has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
59
// th vectors of the query, and so on.
59
- const scalar_t* q_ptr = q + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
60
+ // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
61
+ const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
62
Q_vec q_vecs[NUM_VECS_PER_THREAD];
63
#pragma unroll
64
for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
@@ -264,7 +266,8 @@ __global__ void single_query_cached_kv_attention_kernel(
266
scale, \
267
block_tables_ptr, \
268
context_lens_ptr, \
267
- max_num_blocks_per_seq);
269
+ max_num_blocks_per_seq, \
270
+ query_stride);
271
272
// TODO(woosuk): Tune NUM_THREADS.
273
template<
@@ -284,6 +287,7 @@ void single_query_cached_kv_attention_launcher(
287
int num_heads = query.size(1);
288
int head_size = query.size(2);
289
int max_num_blocks_per_seq = block_tables.size(1);
290
+ int query_stride = query.stride(0);
291
292
T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
293
T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
@@ -333,13 +337,13 @@ void single_query_cached_kv_attention_launcher(
337
}
338
339
void single_query_cached_kv_attention(
336
- torch::Tensor& out,
337
- torch::Tensor& query,
338
- torch::Tensor& key_cache,
339
- torch::Tensor& value_cache,
340
+ torch::Tensor& out, // [num_seqs, num_heads, head_size]
341
+ torch::Tensor& query, // [num_seqs, num_heads, head_size]
342
+ torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
343
+ torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
344
float scale,
341
- torch::Tensor& block_tables,
342
- torch::Tensor& context_lens,
345
+ torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
346
+ torch::Tensor& context_lens, // [num_seqs]
347
int block_size,
348
int max_context_len) {
349
// TODO(woosuk): Support BF16.
csrc/cache_kernels.cu
+17
-8
@@ -81,6 +81,8 @@ __global__ void reshape_and_cache_kernel(
81
scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
82
scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
83
const int* __restrict__ slot_mapping, // [num_tokens]
84
+ const int key_stride,
85
+ const int value_stride,
86
const int num_heads,
87
const int head_size,
88
const int block_size,
@@ -92,7 +94,8 @@ __global__ void reshape_and_cache_kernel(
94
95
const int n = num_heads * head_size;
96
for (int i = threadIdx.x; i < n; i += blockDim.x) {
95
- const int src_idx = token_idx * n + i;
97
+ const int src_key_idx = token_idx * key_stride + i;
98
+ const int src_value_idx = token_idx * value_stride + i;
99
100
const int head_idx = i / head_size;
101
const int head_offset = i % head_size;
@@ -108,25 +111,29 @@ __global__ void reshape_and_cache_kernel(
111
+ head_idx * head_size * block_size
112
+ head_offset * block_size
113
+ block_offset;
111
- key_cache[tgt_key_idx] = __ldg(&key[src_idx]);
112
- value_cache[tgt_value_idx] = __ldg(&value[src_idx]);
114
+ key_cache[tgt_key_idx] = __ldg(&key[src_key_idx]);
115
+ value_cache[tgt_value_idx] = __ldg(&value[src_value_idx]);
116
}
117
}
118
119
} // namespace cacheflow
120
121
void reshape_and_cache(
119
- torch::Tensor& key,
120
- torch::Tensor& value,
121
- torch::Tensor& key_cache,
122
- torch::Tensor& value_cache,
123
- torch::Tensor& slot_mapping) {
122
+ torch::Tensor& key, // [num_tokens, num_heads, head_size]
123
+ torch::Tensor& value, // [num_tokens, num_heads, head_size]
124
+ torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
125
+ torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
126
+ torch::Tensor& slot_mapping) // [num_tokens]
127
+{
128
int num_tokens = key.size(0);
129
int num_heads = key.size(1);
130
int head_size = key.size(2);
131
int block_size = key_cache.size(3);
132
int x = key_cache.size(4);
133
134
+ int key_stride = key.stride(0);
135
+ int value_stride = value.stride(0);
136
+
137
dim3 grid(num_tokens);
138
dim3 block(std::min(num_heads * head_size, 512));
139
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
@@ -140,6 +147,8 @@ void reshape_and_cache(
147
key_cache.data_ptr<scalar_t>(),
148
value_cache.data_ptr<scalar_t>(),
149
slot_mapping.data_ptr<int>(),
150
+ key_stride,
151
+ value_stride,
152
num_heads,
153
head_size,
154
block_size,
csrc/pos_encoding.cpp
-2
@@ -1,8 +1,6 @@
1
#include <torch/extension.h>
2
3
void rotary_embedding_neox(
4
- torch::Tensor& out_query,
5
- torch::Tensor& out_key,
4
torch::Tensor& positions,
5
torch::Tensor& query,
6
torch::Tensor& key,
csrc/pos_encoding_kernels.cu
+22
-27
@@ -5,12 +5,11 @@ namespace cacheflow {
5
6
template<typename scalar_t>
7
__global__ void rotary_embedding_neox_kernel(
8
- scalar_t* __restrict__ out_query, // [num_tokens, num_heads, head_size]
9
- scalar_t* __restrict__ out_key, // [num_tokens, num_heads, head_size]
8
const int64_t* __restrict__ positions, // [num_tokens]
11
- const scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
12
- const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
9
+ scalar_t* __restrict__ query, // [num_tokens, num_heads, head_size]
10
+ scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
11
const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, head_size // 2]
12
+ const int stride,
13
const int num_heads,
14
const int head_size) {
15
// Each thread block is responsible for one token.
@@ -19,41 +18,36 @@ __global__ void rotary_embedding_neox_kernel(
18
const scalar_t* cache_ptr = cos_sin_cache + pos * head_size;
19
20
const int embed_dim = head_size / 2;
22
- const int n = num_heads * head_size;
21
+ const int n = num_heads * embed_dim;
22
for (int i = threadIdx.x; i < n; i += blockDim.x) {
24
- const int idx = token_idx * n + i;
23
+ const int head_idx = i / embed_dim;
24
+ const int token_head = token_idx * stride + head_idx * head_size;
25
26
- const int head_idx = i / head_size;
27
- const int head_offset = i % head_size;
28
- const int token_head = token_idx * n + head_idx * head_size;
29
-
30
- const bool is_first_half = head_offset < embed_dim;
31
- const int rot_offset = head_offset % embed_dim;
26
+ const int rot_offset = i % embed_dim;
27
const int x_index = rot_offset;
28
const int y_index = embed_dim + rot_offset;
29
30
+ const int out_x = token_idx * stride + head_idx * head_size + x_index;
31
+ const int out_y = token_idx * stride + head_idx * head_size + y_index;
32
+
33
const scalar_t cos = __ldg(cache_ptr + x_index);
34
const scalar_t sin = __ldg(cache_ptr + y_index);
35
38
- const scalar_t q_x = __ldg(query + token_head + x_index);
39
- const scalar_t q_y = __ldg(query + token_head + y_index);
40
- const scalar_t q_cos = is_first_half ? q_x : q_y;
41
- const scalar_t q_sin = is_first_half ? -q_y : q_x;
42
- out_query[idx] = q_cos * cos + q_sin * sin;
36
+ const scalar_t q_x = query[token_head + x_index];
37
+ const scalar_t q_y = query[token_head + y_index];
38
+ query[out_x] = q_x * cos - q_y * sin;
39
+ query[out_y] = q_y * cos + q_x * sin;
40
44
- const scalar_t k_x = __ldg(key + token_head + x_index);
45
- const scalar_t k_y = __ldg(key + token_head + y_index);
46
- const scalar_t k_cos = is_first_half ? k_x : k_y;
47
- const scalar_t k_sin = is_first_half ? -k_y : k_x;
48
- out_key[idx] = k_cos * cos + k_sin * sin;
41
+ const scalar_t k_x = key[token_head + x_index];
42
+ const scalar_t k_y = key[token_head + y_index];
43
+ key[out_x] = k_x * cos - k_y * sin;
44
+ key[out_y] = k_y * cos + k_x * sin;
45
}
46
}
47
48
} // namespace cacheflow
49
50
void rotary_embedding_neox(
55
- torch::Tensor& out_query, // [num_tokens, num_heads * head_size]
56
- torch::Tensor& out_key, // [num_tokens, num_heads * head_size]
51
torch::Tensor& positions, // [num_tokens]
52
torch::Tensor& query, // [num_tokens, num_heads * head_size]
53
torch::Tensor& key, // [num_tokens, num_heads * head_size]
@@ -62,21 +56,22 @@ void rotary_embedding_neox(
56
int num_tokens = query.size(0);
57
int head_size = cos_sin_cache.size(1);
58
int num_heads = query.size(1) / head_size;
59
+ int stride = query.stride(0);
60
+ TORCH_CHECK(stride == key.stride(0));
61
62
dim3 grid(num_tokens);
67
- dim3 block(std::min(num_heads * head_size, 512));
63
+ dim3 block(std::min(num_heads * head_size / 2, 512));
64
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
65
AT_DISPATCH_FLOATING_TYPES_AND_HALF(
66
query.scalar_type(),
67
"rotary_embedding_neox",
68
[&] {
69
cacheflow::rotary_embedding_neox_kernel<scalar_t><<<grid, block, 0, stream>>>(
74
- out_query.data_ptr<scalar_t>(),
75
- out_key.data_ptr<scalar_t>(),
70
positions.data_ptr<int64_t>(),
71
query.data_ptr<scalar_t>(),
72
key.data_ptr<scalar_t>(),
73
cos_sin_cache.data_ptr<scalar_t>(),
74
+ stride,
75
num_heads,
76
head_size);
77
});
setup.py
+7
@@ -39,6 +39,13 @@ layernorm_extension = cpp_extension.CUDAExtension(
39
)
40
ext_modules.append(layernorm_extension)
41
42
+activation_extension = cpp_extension.CUDAExtension(
43
+ name='cacheflow.activation_ops',
44
+ sources=['csrc/activation.cpp', 'csrc/activation_kernels.cu'],
45
+ extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
46
+)
47
+ext_modules.append(activation_extension)
48
+
49
setuptools.setup(
50
name='cacheflow',
51
ext_modules=ext_modules,
tests/kernels/activation.py
new
+30
@@ -0,0 +1,30 @@
1
+import torch
2
+import torch.nn.functional as F
3
+
4
+from cacheflow import activation_ops
5
+
6
+
7
+def ref_silu_and_mul(x: torch.Tensor) -> torch.Tensor:
8
+ x1, x2 = x.chunk(chunks=2, dim=1)
9
+ return F.silu(x1) * x2
10
+
11
+
12
+@torch.inference_mode()
13
+def test_silu_and_mul(
14
+ num_tokens: int,
15
+ d: int,
16
+ dtype: torch.dtype,
17
+) -> None:
18
+ x = torch.randn(num_tokens, 2 * d, dtype=dtype, device='cuda')
19
+ out = torch.empty(num_tokens, d, dtype=dtype, device='cuda')
20
+ activation_ops.silu_and_mul(out, x)
21
+ ref_out = ref_silu_and_mul(x)
22
+ assert torch.allclose(out, ref_out, atol=1e-5, rtol=1e-5)
23
+
24
+
25
+if __name__ == '__main__':
26
+ for dtype in [torch.half, torch.float]:
27
+ for num_tokens in [7, 83, 2048]:
28
+ for d in [512, 4096, 13824]:
29
+ print(f'Testing dtype={dtype}, num_tokens={num_tokens}, d={d}')
30
+ test_silu_and_mul(num_tokens, d, dtype)
tests/kernels/attention.py
+31
-15
@@ -1,7 +1,7 @@
1
import random
2
from typing import List, Optional
3
4
-from flash_attn.flash_attention import FlashAttention
4
+from flash_attn.flash_attn_interface import _flash_attn_forward
5
import torch
6
7
from cacheflow import attention_ops
@@ -105,8 +105,9 @@ def test_single_query_cached_kv_attention(
105
num_blocks: int,
106
dtype: torch.dtype,
107
) -> None:
108
- query = torch.randn(
109
- num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
108
+ qkv = torch.randn(
109
+ num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
110
+ query, _, _ = qkv.unbind(dim=1)
111
x = 16 // torch.tensor([], dtype=dtype).element_size()
112
key_block_shape = (num_heads, head_size // x, block_size, x)
113
key_cache = torch.randn(
@@ -115,6 +116,11 @@ def test_single_query_cached_kv_attention(
116
value_cache = torch.randn(
117
size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
118
119
+ # Adjust the range of the values to reduce precision errors.
120
+ query = query / (head_size ** 0.5)
121
+ key_cache = key_cache / (head_size ** 0.5)
122
+ value_cache = value_cache / (head_size ** 0.5)
123
+
124
context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
125
max_context_len = max(context_lens)
126
context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
@@ -130,7 +136,8 @@ def test_single_query_cached_kv_attention(
136
block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
137
138
scale = float(1.0 / (head_size ** 0.5))
133
- output = torch.empty_like(query)
139
+ output = torch.empty(
140
+ num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
141
attention_ops.single_query_cached_kv_attention(
142
output,
143
query,
@@ -175,19 +182,28 @@ def test_multi_query_kv_attention(
182
cu_seq_lens = torch.tensor(cu_seq_lens, dtype=torch.int, device='cuda')
183
184
scale = float(1.0 / (head_size ** 0.5))
178
- query = torch.randn(
185
+ qkv = torch.randn(
186
+ num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
187
+ # Adjust the range of the values to reduce precision errors.
188
+ qkv = qkv / (head_size ** 0.5)
189
+
190
+ query, key, value = qkv.unbind(dim=1)
191
+ output = torch.empty(
192
num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
180
- key = torch.rand_like(query)
181
- value = torch.rand_like(query)
182
-
183
- qkv = torch.stack([query, key, value], dim=1)
184
- flash_attn = FlashAttention(softmax_scale=scale)
185
- output = flash_attn(
186
- qkv,
187
- cu_seqlens=cu_seq_lens,
188
- max_s=max_seq_len,
193
+ _flash_attn_forward(
194
+ query,
195
+ key,
196
+ value,
197
+ output,
198
+ cu_seq_lens,
199
+ cu_seq_lens,
200
+ max_seq_len,
201
+ max_seq_len,
202
+ dropout_p=0.0,
203
+ softmax_scale=scale,
204
causal=True,
190
- )[0]
205
+ return_softmax=False,
206
+ )
207
208
cu_seq_lens = cu_seq_lens.cpu().tolist()
209
ref_output = ref_multi_query_kv_attention(
tests/kernels/cache.py
+5
-5
@@ -17,10 +17,10 @@ def test_reshape_and_cache(
17
slot_mapping = random.sample(range(num_slots), num_tokens)
18
slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
19
20
- kv_shape = (num_tokens, num_heads, head_size)
21
- key = torch.randn(size=kv_shape, dtype=dtype, device='cuda')
22
- value = torch.randn(size=kv_shape, dtype=dtype, device='cuda')
23
-
20
+ qkv = torch.randn(
21
+ num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
22
+ _, key, value = qkv.unbind(dim=1)
23
+
24
x = 16 // torch.tensor([], dtype=dtype).element_size()
25
key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
26
key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
@@ -35,7 +35,7 @@ def test_reshape_and_cache(
35
36
for i in range(num_tokens):
37
reshaped_key = key.reshape(num_tokens, num_heads, head_size // x, x)
38
- block_idx = slot_mapping[i] // block_size
38
+ block_idx = torch.div(slot_mapping[i], block_size, rounding_mode='floor')
39
block_offset = slot_mapping[i] % block_size
40
cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
41
cloned_value_cache[block_idx, :, :, block_offset] = value[i]
tests/kernels/pos_encoding.py
+4
-6
@@ -85,15 +85,13 @@ def test_rotary_embedding_neox(
85
cos_sin_cache = torch.cat((cos, sin), dim=-1)
86
cos_sin_cache = cos_sin_cache.to(dtype=dtype, device='cuda')
87
88
- # Run the kernel.
89
- out_query = torch.empty_like(query)
90
- out_key = torch.empty_like(key)
88
+ # Run the kernel. The kernel is in-place, so we need to clone the inputs.
89
+ out_query = query.clone()
90
+ out_key = key.clone()
91
pos_encoding_ops.rotary_embedding_neox(
92
+ positions,
93
out_query,
94
out_key,
94
- positions,
95
- query,
96
- key,
95
cos_sin_cache,
96
)
97