A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Implement `single_query_cached_kv_attention` kernel (#3)
Woosuk Kwon committed
Mar 1, 2023 at 15:02 UTC
0deacbce6e96a1af5885babc4e470ce2a0cecf95
12 files changed
+2140
-60
cacheflow/master/block_manager.py
+3
-1
@@ -15,7 +15,9 @@ class BlockManager:
15
block_size: int,
16
num_blocks: int,
17
) -> None:
18
- assert block_size in [8, 16, 32]
18
+ if block_size not in [8, 16]:
19
+ raise ValueError(f'Unsupported block size: {block_size}'
20
+ 'The block size must be either 8 or 16.')
21
self.device = device
22
self.block_size = block_size
23
self.num_blocks = num_blocks
cacheflow/models/attention.py
+19
-35
@@ -3,7 +3,8 @@ from typing import List, Optional
3
import torch
4
import torch.nn as nn
5
6
-from cacheflow import ops
6
+from cacheflow import attention_ops
7
+from cacheflow import cache_ops
8
from cacheflow.models import InputMetadata
9
10
@@ -11,7 +12,7 @@ class OPTCacheFlowAttention(nn.Module):
12
13
def __init__(self, scale: float) -> None:
14
super().__init__()
14
- self.scale = scale
15
+ self.scale = float(scale)
16
17
def _masked_attention(
18
self,
@@ -57,38 +58,21 @@ class OPTCacheFlowAttention(nn.Module):
58
output: torch.Tensor, # [num_generation_tokens, num_heads, head_size]
59
query: torch.Tensor, # [num_generation_tokens, num_heads, head_size]
60
key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
60
- value_cache: torch.Tensor, # [num_blocks, num_heads, block_size, head_size]
61
+ value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
62
input_metadata: InputMetadata,
63
) -> None:
63
- num_heads = value_cache.shape[1]
64
- head_size = value_cache.shape[3]
65
- block_size = value_cache.shape[2]
66
- block_tables = input_metadata.block_tables
67
-
68
- # FIXME(woosuk): Replace the following with a custom op.
69
- for i in range(input_metadata.num_generation_tokens):
70
- q = query[i].unsqueeze(0)
71
- block_table = block_tables[i]
72
- context_len = int(input_metadata.context_lens[i])
73
-
74
- keys = []
75
- values = []
76
- for j in range(context_len):
77
- block_number = int(block_table[j // block_size])
78
- block_offset = j % block_size
79
-
80
- k = key_cache[block_number, :, :, block_offset, :]
81
- k = k.reshape(num_heads, head_size)
82
- keys.append(k)
83
-
84
- v = value_cache[block_number, :, block_offset, :]
85
- values.append(v)
86
- keys = torch.stack(keys, dim=0)
87
- values = torch.stack(values, dim=0)
88
-
89
- out = self._masked_attention(q, keys, values)
90
- out = out.view(num_heads, head_size)
91
- output[i].copy_(out, non_blocking=True)
64
+ block_size = value_cache.shape[3]
65
+ attention_ops.single_query_cached_kv_attention(
66
+ output,
67
+ query,
68
+ key_cache,
69
+ value_cache,
70
+ self.scale,
71
+ input_metadata.block_tables,
72
+ input_metadata.context_lens,
73
+ block_size,
74
+ input_metadata.max_context_len,
75
+ )
76
77
def forward(
78
self,
@@ -96,7 +80,7 @@ class OPTCacheFlowAttention(nn.Module):
80
key: torch.Tensor, # [num_tokens, num_heads * head_size]
81
value: torch.Tensor, # [num_tokens, num_heads * head_size]
82
key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
99
- value_cache: torch.Tensor, # [num_blocks, num_heads, block_size, head_size]
83
+ value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
84
input_metadata: InputMetadata,
85
cache_event: Optional[torch.cuda.Event],
86
) -> torch.Tensor: # [num_tokens, num_heads * head_size]
@@ -110,7 +94,7 @@ class OPTCacheFlowAttention(nn.Module):
94
95
# Reshape the input tensors.
96
num_heads = value_cache.shape[1]
113
- head_size = value_cache.shape[3]
97
+ head_size = value_cache.shape[2]
98
query = query.view(-1, num_heads, head_size)
99
key = key.view(-1, num_heads, head_size)
100
value = value.view(-1, num_heads, head_size)
@@ -125,7 +109,7 @@ class OPTCacheFlowAttention(nn.Module):
109
cache_event.wait()
110
111
# Reshape the keys and values and store them in the cache.
128
- ops.reshape_and_cache(
112
+ cache_ops.reshape_and_cache(
113
key, value, key_cache, value_cache, input_metadata.slot_mapping)
114
115
if input_metadata.num_generation_tokens > 0:
cacheflow/worker/cache_engine.py
+12
-8
@@ -1,7 +1,7 @@
1
from typing import Dict, List, Tuple
2
3
import torch
4
-from cacheflow import ops
4
+from cacheflow import cache_ops
5
6
KVCache = Tuple[torch.Tensor, torch.Tensor]
7
@@ -57,20 +57,22 @@ class CacheEngine:
57
def get_value_block_shape(self) -> Tuple[int, int, int]:
58
return (
59
self.num_heads,
60
- self.block_size,
60
self.head_size,
61
+ self.block_size,
62
)
63
64
def allocate_gpu_cache(self) -> List[KVCache]:
65
gpu_cache: List[KVCache] = []
66
+ key_block_shape = self.get_key_block_shape()
67
+ value_block_shape = self.get_value_block_shape()
68
for _ in range(self.num_layers):
69
key_blocks = torch.empty(
68
- size=(self.num_gpu_blocks, *self.get_key_block_shape()),
70
+ size=(self.num_gpu_blocks, *key_block_shape),
71
dtype=self.dtype,
72
device=self.gpu_id,
73
)
74
value_blocks = torch.empty(
73
- size=(self.num_gpu_blocks, *self.get_value_block_shape()),
75
+ size=(self.num_gpu_blocks, *value_block_shape),
76
dtype=self.dtype,
77
device=self.gpu_id,
78
)
@@ -79,14 +81,16 @@ class CacheEngine:
81
82
def allocate_cpu_cache(self) -> List[KVCache]:
83
cpu_cache: List[KVCache] = []
84
+ key_block_shape = self.get_key_block_shape()
85
+ value_block_shape = self.get_value_block_shape()
86
for _ in range(self.num_layers):
87
key_blocks = torch.empty(
84
- size=(self.num_cpu_blocks, *self.get_key_block_shape()),
88
+ size=(self.num_cpu_blocks, *key_block_shape),
89
dtype=self.dtype,
90
pin_memory=True,
91
)
92
value_blocks = torch.empty(
89
- size=(self.num_cpu_blocks, *self.get_value_block_shape()),
93
+ size=(self.num_cpu_blocks, *value_block_shape),
94
dtype=self.dtype,
95
pin_memory=True,
96
)
@@ -104,10 +108,10 @@ class CacheEngine:
108
src_key_cache, src_value_cache = src[i]
109
dst_key_cache, dst_value_cache = dst[i]
110
# Copy the key blocks.
107
- ops.copy_cache_blocks(
111
+ cache_ops.copy_cache_blocks(
112
src_key_cache, dst_key_cache, src_to_dst)
113
# Copy the value blocks.
110
- ops.copy_cache_blocks(
114
+ cache_ops.copy_cache_blocks(
115
src_value_cache, dst_value_cache, src_to_dst)
116
event = self.events[i]
117
event.record(stream=self.cache_stream)
cacheflow/worker/worker.py
+1
-1
@@ -118,7 +118,7 @@ class Worker:
118
_pad_to_max(block_table, max_num_blocks_per_seq)
119
for block_table in generation_block_tables]
120
block_tables_tensor = torch.tensor(
121
- padded_block_tables, dtype=int, device=self.device)
121
+ padded_block_tables, dtype=torch.int, device=self.device)
122
123
input_metadata = InputMetadata(
124
seq_ids=prompt_seq_ids + generation_seq_ids,
csrc/attention.cpp
new
+19
@@ -0,0 +1,19 @@
1
+#include <torch/extension.h>
2
+
3
+void single_query_cached_kv_attention(
4
+ torch::Tensor& out,
5
+ torch::Tensor& query,
6
+ torch::Tensor& key_cache,
7
+ torch::Tensor& value_cache,
8
+ float scale,
9
+ torch::Tensor& block_tables,
10
+ torch::Tensor& context_lens,
11
+ int block_size,
12
+ int max_context_len);
13
+
14
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
15
+ m.def(
16
+ "single_query_cached_kv_attention",
17
+ &single_query_cached_kv_attention,
18
+ "Compute the attention between an input query and the cached key/value tensors");
19
+}
csrc/attention_kernels.cu
new
+400
@@ -0,0 +1,400 @@
1
+#include <torch/extension.h>
2
+#include <ATen/cuda/CUDAContext.h>
3
+
4
+#include "attention_utils.h"
5
+#include "cuda_primitives.h"
6
+
7
+#include <algorithm>
8
+
9
+#define WARP_SIZE 32
10
+
11
+namespace cacheflow {
12
+
13
+// Grid: (num_heads, num_seqs).
14
+template<
15
+ typename scalar_t,
16
+ int HEAD_SIZE,
17
+ int BLOCK_SIZE,
18
+ int NUM_THREADS>
19
+__global__ void single_query_cached_kv_attention_kernel(
20
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
21
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
22
+ const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
23
+ const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
24
+ const float scale,
25
+ const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
26
+ const int* __restrict__ context_lens, // [num_seqs]
27
+ const int max_num_blocks_per_seq) {
28
+ constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
29
+ constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
30
+ const int thread_idx = threadIdx.x;
31
+ const int warp_idx = thread_idx / WARP_SIZE;
32
+ const int lane = thread_idx % WARP_SIZE;
33
+
34
+ const int head_idx = blockIdx.x;
35
+ const int num_heads = gridDim.x;
36
+ const int seq_idx = blockIdx.y;
37
+
38
+ // A vector type to store a part of a key or a query.
39
+ // The vector size is configured in such a way that the threads in a thread group
40
+ // fetch or comput 16 bytes at a time.
41
+ // For example, if the size of a thread group is 4 and the data type is half,
42
+ // then the vector size is 16 / (4 * sizeof(half)) == 2.
43
+ constexpr int VEC_SIZE = 16 / (THREAD_GROUP_SIZE * sizeof(scalar_t));
44
+ using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
45
+ using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
46
+
47
+ constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
48
+ constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
49
+
50
+ const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
51
+ const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
52
+
53
+ // Load the query to registers.
54
+ // Each thread in a thread group has a different part of the query.
55
+ // For example, if the the thread group size is 4, then the first thread in the group
56
+ // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
57
+ // th vectors of the query, and so on.
58
+ const scalar_t* q_ptr = q + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
59
+ Q_vec q_vecs[NUM_VECS_PER_THREAD];
60
+#pragma unroll
61
+ for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
62
+ const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
63
+ q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
64
+ }
65
+
66
+ // Memory planning.
67
+ extern __shared__ char shared_mem[];
68
+ // NOTE(woosuk): We use FP32 logits and accumulation.
69
+ float *logits = reinterpret_cast<float*>(shared_mem);
70
+ // Workspace for reduction.
71
+ __shared__ float red_smem[2 * NUM_WARPS];
72
+
73
+ // x == THREAD_GROUP_SIZE * VEC_SIZE
74
+ // Each thread group fetches x elements from the key at a time.
75
+ constexpr int x = 16 / sizeof(scalar_t);
76
+ float qk_max = -FLT_MAX;
77
+
78
+ const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
79
+ const int context_len = context_lens[seq_idx];
80
+ const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
81
+
82
+ // Iterate over the key blocks.
83
+ // Each warp fetches a block of keys for each iteration.
84
+ // Each thread group in a warp fetches a key from the block, and computes
85
+ // dot product with the query.
86
+ for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
87
+ const int physical_block_number = block_table[block_idx];
88
+ const int physical_block_offset = thread_group_idx % BLOCK_SIZE;
89
+ const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
90
+
91
+ // Load a key to registers.
92
+ // Each thread in a thread group has a different part of the key.
93
+ // For example, if the the thread group size is 4, then the first thread in the group
94
+ // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
95
+ // vectors of the key, and so on.
96
+ K_vec k_vecs[NUM_VECS_PER_THREAD];
97
+#pragma unroll
98
+ for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
99
+ const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
100
+ + head_idx * HEAD_SIZE * BLOCK_SIZE
101
+ + physical_block_offset * x;
102
+ const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
103
+ const int offset1 = (vec_idx * VEC_SIZE) / x;
104
+ const int offset2 = (vec_idx * VEC_SIZE) % x;
105
+ k_vecs[i] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
106
+ }
107
+
108
+ // Compute dot product.
109
+ // This includes a reduction across the threads in the same thread group.
110
+ const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
111
+ const bool mask = token_idx >= context_len;
112
+
113
+ if (thread_group_offset == 0) {
114
+ // Store the partial reductions to shared memory.
115
+ // NOTE(woosuk): It is required to zero out the masked logits.
116
+ logits[token_idx] = mask ? 0.f : qk;
117
+ // Update the max value.
118
+ qk_max = mask ? qk_max : fmaxf(qk_max, qk);
119
+ }
120
+ }
121
+
122
+ // Perform reduction across the threads in the same warp to get the
123
+ // max qk value for each "warp" (not across the thread block yet).
124
+ // The 0-th thread of each thread group already has its max qk value.
125
+#pragma unroll
126
+ for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
127
+ qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
128
+ }
129
+ if (lane == 0) {
130
+ red_smem[warp_idx] = qk_max;
131
+ }
132
+ __syncthreads();
133
+
134
+ // TODO(woosuk): Refactor this part.
135
+ // Get the max qk value for the sequence.
136
+ qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
137
+#pragma unroll
138
+ for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
139
+ qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
140
+ }
141
+ // Broadcast the max qk value to all threads.
142
+ qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
143
+
144
+ // Get the sum of the exp values.
145
+ float exp_sum = 0.f;
146
+ for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
147
+ float val = __expf(logits[i] - qk_max);
148
+ logits[i] = val;
149
+ exp_sum += val;
150
+ }
151
+ exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
152
+
153
+ // Compute softmax.
154
+ const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
155
+ for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
156
+ logits[i] *= inv_sum;
157
+ }
158
+ __syncthreads();
159
+
160
+ // Each thread will fetch 16 bytes from the value cache at a time.
161
+ constexpr int V_VEC_SIZE = 16 / sizeof(scalar_t);
162
+ using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
163
+ using L_vec = typename FloatVec<V_vec>::Type;
164
+
165
+ constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
166
+ constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
167
+ constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
168
+
169
+ float accs[NUM_ROWS_PER_THREAD];
170
+#pragma unroll
171
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
172
+ accs[i] = 0.f;
173
+ }
174
+
175
+ for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
176
+ const int physical_block_number = block_table[block_idx];
177
+ const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
178
+ const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
179
+ L_vec logits_vec = *reinterpret_cast<L_vec*>(logits + token_idx);
180
+
181
+ const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
182
+ + head_idx * HEAD_SIZE * BLOCK_SIZE;
183
+#pragma unroll
184
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
185
+ const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
186
+ if (row_idx < HEAD_SIZE) {
187
+ const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
188
+ V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
189
+ accs[i] += dot(logits_vec, cast_to_float(v_vec));
190
+ }
191
+ }
192
+ }
193
+
194
+ // Perform reduction within each warp.
195
+#pragma unroll
196
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
197
+ float acc = accs[i];
198
+#pragma unroll
199
+ for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
200
+ acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
201
+ }
202
+ accs[i] = acc;
203
+ }
204
+
205
+ // NOTE(woosuk): A barrier is required because the shared memory space for logits
206
+ // is reused for the output.
207
+ __syncthreads();
208
+
209
+ // Perform reduction across warps.
210
+ float* out_smem = reinterpret_cast<float*>(shared_mem);
211
+#pragma unroll
212
+ for (int i = NUM_WARPS; i > 1; i /= 2) {
213
+ int mid = i / 2;
214
+ // Upper warps write to shared memory.
215
+ if (warp_idx >= mid && warp_idx < i) {
216
+ float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
217
+#pragma unroll
218
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
219
+ const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
220
+ if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
221
+ dst[row_idx] = accs[i];
222
+ }
223
+ }
224
+ }
225
+ __syncthreads();
226
+
227
+ // Lower warps update the output.
228
+ if (warp_idx < mid) {
229
+ const float* src = &out_smem[warp_idx * HEAD_SIZE];
230
+#pragma unroll
231
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
232
+ const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
233
+ if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
234
+ accs[i] += src[row_idx];
235
+ }
236
+ }
237
+ }
238
+ __syncthreads();
239
+ }
240
+
241
+ // Write the final output.
242
+ if (warp_idx == 0) {
243
+ scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
244
+#pragma unroll
245
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
246
+ const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
247
+ if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
248
+ convert_from_float(*(out_ptr + row_idx), accs[i]);
249
+ }
250
+ }
251
+ }
252
+}
253
+
254
+} // namespace cacheflow
255
+
256
+#define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
257
+ cacheflow::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
258
+ <<<grid, block, shared_mem_size, stream>>>( \
259
+ out_ptr, \
260
+ query_ptr, \
261
+ key_cache_ptr, \
262
+ value_cache_ptr, \
263
+ scale, \
264
+ block_tables_ptr, \
265
+ context_lens_ptr, \
266
+ max_num_blocks_per_seq);
267
+
268
+// TODO(woosuk): Tune NUM_THREADS.
269
+template<
270
+ typename T,
271
+ int BLOCK_SIZE,
272
+ int NUM_THREADS = 128>
273
+void single_query_cached_kv_attention_launcher(
274
+ torch::Tensor& out,
275
+ torch::Tensor& query,
276
+ torch::Tensor& key_cache,
277
+ torch::Tensor& value_cache,
278
+ float scale,
279
+ torch::Tensor& block_tables,
280
+ torch::Tensor& context_lens,
281
+ int max_context_len) {
282
+ int num_seqs = query.size(0);
283
+ int num_heads = query.size(1);
284
+ int head_size = query.size(2);
285
+ int max_num_blocks_per_seq = block_tables.size(1);
286
+
287
+ T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
288
+ T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
289
+ T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
290
+ T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
291
+ int* block_tables_ptr = block_tables.data_ptr<int>();
292
+ int* context_lens_ptr = context_lens.data_ptr<int>();
293
+
294
+ constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
295
+ int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
296
+ int logits_size = padded_max_context_len * sizeof(float);
297
+ int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
298
+ int shared_mem_size = std::max(logits_size, outputs_size);
299
+
300
+ dim3 grid(num_heads, num_seqs);
301
+ dim3 block(NUM_THREADS);
302
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
303
+ switch (head_size) {
304
+ case 32:
305
+ LAUNCH_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
306
+ break;
307
+ case 64:
308
+ LAUNCH_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
309
+ break;
310
+ case 80:
311
+ LAUNCH_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
312
+ break;
313
+ case 96:
314
+ LAUNCH_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
315
+ break;
316
+ case 128:
317
+ LAUNCH_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
318
+ break;
319
+ case 160:
320
+ LAUNCH_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
321
+ break;
322
+ case 192:
323
+ LAUNCH_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
324
+ break;
325
+ case 256:
326
+ LAUNCH_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
327
+ break;
328
+ default:
329
+ assert(false);
330
+ break;
331
+ }
332
+}
333
+
334
+void single_query_cached_kv_attention(
335
+ torch::Tensor& out,
336
+ torch::Tensor& query,
337
+ torch::Tensor& key_cache,
338
+ torch::Tensor& value_cache,
339
+ float scale,
340
+ torch::Tensor& block_tables,
341
+ torch::Tensor& context_lens,
342
+ int block_size,
343
+ int max_context_len) {
344
+ // TODO(woosuk): Support BF16.
345
+ if (query.element_size() == 2) {
346
+ // Half.
347
+ if (block_size == 8) {
348
+ single_query_cached_kv_attention_launcher<uint16_t, 8>(
349
+ out,
350
+ query,
351
+ key_cache,
352
+ value_cache,
353
+ scale,
354
+ block_tables,
355
+ context_lens,
356
+ max_context_len);
357
+ } else if (block_size == 16) {
358
+ single_query_cached_kv_attention_launcher<uint16_t, 16>(
359
+ out,
360
+ query,
361
+ key_cache,
362
+ value_cache,
363
+ scale,
364
+ block_tables,
365
+ context_lens,
366
+ max_context_len);
367
+ } else {
368
+ assert(false);
369
+ }
370
+ } else if (query.element_size() == 4) {
371
+ // Float.
372
+ if (block_size == 8) {
373
+ single_query_cached_kv_attention_launcher<float, 8>(
374
+ out,
375
+ query,
376
+ key_cache,
377
+ value_cache,
378
+ scale,
379
+ block_tables,
380
+ context_lens,
381
+ max_context_len);
382
+ } else if (block_size == 16) {
383
+ single_query_cached_kv_attention_launcher<float, 16>(
384
+ out,
385
+ query,
386
+ key_cache,
387
+ value_cache,
388
+ scale,
389
+ block_tables,
390
+ context_lens,
391
+ max_context_len);
392
+ } else {
393
+ assert(false);
394
+ }
395
+ } else {
396
+ assert(false);
397
+ }
398
+}
399
+
400
+#undef WARP_SIZE
csrc/attention_utils.h
new
+204
@@ -0,0 +1,204 @@
1
+#pragma once
2
+
3
+#include "cuda_primitives.h"
4
+
5
+#include <float.h>
6
+#include <type_traits>
7
+
8
+#define MMHA_USE_FP32_ACUM_FOR_FMA
9
+#define MMHA_USE_FP32_ACUM_FOR_OUT
10
+
11
+namespace cacheflow {
12
+
13
+// A vector type to store Q, K, V elements.
14
+template<typename T, int VEC_SIZE>
15
+struct Vec {};
16
+template<>
17
+struct Vec<float, 1> {
18
+ using Type = float;
19
+};
20
+template<>
21
+struct Vec<float, 2> {
22
+ using Type = float2;
23
+};
24
+template<>
25
+struct Vec<float, 4> {
26
+ using Type = float4;
27
+};
28
+template<>
29
+struct Vec<uint16_t, 1> {
30
+ using Type = uint16_t;
31
+};
32
+template<>
33
+struct Vec<uint16_t, 2> {
34
+ using Type = uint32_t;
35
+};
36
+template<>
37
+struct Vec<uint16_t, 4> {
38
+ using Type = uint2;
39
+};
40
+template<>
41
+struct Vec<uint16_t, 8> {
42
+ using Type = uint4;
43
+};
44
+
45
+template<typename T>
46
+struct FloatVec {};
47
+template<>
48
+struct FloatVec<float> {
49
+ using Type = float;
50
+};
51
+template<>
52
+struct FloatVec<float2> {
53
+ using Type = float2;
54
+};
55
+template<>
56
+struct FloatVec<float4> {
57
+ using Type = float4;
58
+};
59
+template<>
60
+struct FloatVec<uint16_t> {
61
+ using Type = float;
62
+};
63
+template<>
64
+struct FloatVec<uint32_t> {
65
+ using Type = float2;
66
+};
67
+template<>
68
+struct FloatVec<uint2> {
69
+ using Type = Float4_;
70
+};
71
+template<>
72
+struct FloatVec<uint4> {
73
+ using Type = Float8_;
74
+};
75
+
76
+template<int THREADS_PER_KEY, typename K_vec, int N>
77
+inline __device__ float qk_dot_(const K_vec (&q)[N], const K_vec (&k)[N])
78
+{
79
+ using K_vec_acum = typename FloatVec<K_vec>::Type;
80
+ // Compute the parallel products for Q*K^T (treat vector lanes separately).
81
+ K_vec_acum qk_vec = mul<K_vec_acum, K_vec, K_vec>(q[0], k[0]);
82
+#pragma unroll
83
+ for (int ii = 1; ii < N; ++ii) {
84
+ qk_vec = fma(q[ii], k[ii], qk_vec);
85
+ }
86
+
87
+ // Finalize the reduction across lanes.
88
+ float qk = sum(qk_vec);
89
+#pragma unroll
90
+ for (int mask = THREADS_PER_KEY / 2; mask >= 1; mask /= 2) {
91
+ qk += __shfl_xor_sync(uint32_t(-1), qk, mask);
92
+ }
93
+ return qk;
94
+}
95
+
96
+////////////////////////////////////////////////////////////////////////////////////////////////////
97
+
98
+template<typename T, int THREADS_PER_KEY>
99
+struct Qk_dot {
100
+ template<typename K_vec, int N>
101
+ static inline __device__ float dot(const K_vec (&q)[N], const K_vec (&k)[N])
102
+ {
103
+ return qk_dot_<THREADS_PER_KEY>(q, k);
104
+ }
105
+};
106
+
107
+////////////////////////////////////////////////////////////////////////////////////////////////////
108
+
109
+inline __device__ float4 hmma_fp32(const uint2& a, uint32_t b)
110
+{
111
+ float4 c;
112
+ float zero = 0.f;
113
+ asm volatile("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 \n"
114
+ " {%0, %1, %2, %3}, \n"
115
+ " {%4, %5}, \n"
116
+ " {%6}, \n"
117
+ " {%7, %7, %7, %7}; \n"
118
+
119
+ : "=f"(c.x), "=f"(c.y), "=f"(c.z), "=f"(c.w)
120
+ : "r"(a.x) "r"(a.y), "r"(b), "f"(zero));
121
+ return c;
122
+}
123
+
124
+////////////////////////////////////////////////////////////////////////////////////////////////////
125
+
126
+template<int N>
127
+inline __device__ float qk_hmma_dot_(const uint32_t (&q)[N], const uint32_t (&k)[N])
128
+{
129
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 750
130
+ using K_vec_acum = typename FloatVec<uint32_t>::Type;
131
+ K_vec_acum qk_vec = mul<K_vec_acum, uint32_t, uint32_t>(q[0], k[0]);
132
+#pragma unroll
133
+ for (int ii = 1; ii < N; ++ii) {
134
+ qk_vec = fma(q[ii], k[ii], qk_vec);
135
+ }
136
+#ifdef MMHA_USE_FP32_ACUM_FOR_FMA
137
+ uint32_t qk_vec_ = float2_to_half2(qk_vec);
138
+ return hmma_fp32(make_uint2(qk_vec_, 0u), 0x3c003c00u).x;
139
+#else
140
+ return hmma_fp32(make_uint2(qk_vec, 0u), 0x3c003c00u).x;
141
+#endif
142
+#else
143
+ return 0.f;
144
+#endif
145
+}
146
+
147
+////////////////////////////////////////////////////////////////////////////////////////////////////
148
+
149
+template<>
150
+struct Qk_dot<uint16_t, 4> {
151
+ template<int N>
152
+ static inline __device__ float dot(const uint32_t (&q)[N], const uint32_t (&k)[N])
153
+ {
154
+#if __CUDA_ARCH__ >= 750 && defined(MMHA_USE_HMMA_FOR_REDUCTION)
155
+ return qk_hmma_dot_(q, k);
156
+#else
157
+ return qk_dot_<4>(q, k);
158
+#endif // defined MMHA_USE_HMMA_FOR_REDUCTION
159
+ }
160
+};
161
+
162
+////////////////////////////////////////////////////////////////////////////////////////////////////
163
+
164
+template<int WARPS_PER_BLOCK, int WARP_SIZE = 32>
165
+inline __device__ float block_sum(float* red_smem, float sum)
166
+{
167
+
168
+ // Decompose the thread index into warp / lane.
169
+ int warp = threadIdx.x / WARP_SIZE;
170
+ int lane = threadIdx.x % WARP_SIZE;
171
+
172
+ // Compute the sum per warp.
173
+#pragma unroll
174
+ for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
175
+ sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
176
+ }
177
+
178
+ // Warp leaders store the data to shared memory.
179
+ if (lane == 0) {
180
+ red_smem[warp] = sum;
181
+ }
182
+
183
+ // Make sure the data is in shared memory.
184
+ __syncthreads();
185
+
186
+ // The warps compute the final sums.
187
+ if (lane < WARPS_PER_BLOCK) {
188
+ sum = red_smem[lane];
189
+ }
190
+
191
+// Parallel reduction inside the warp.
192
+#pragma unroll
193
+ for (int mask = WARPS_PER_BLOCK / 2; mask >= 1; mask /= 2) {
194
+ sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
195
+ }
196
+
197
+ // Broadcast to other threads.
198
+ return __shfl_sync(uint32_t(-1), sum, 0);
199
+}
200
+
201
+} // namespace cacheflow
202
+
203
+#undef MMHA_USE_FP32_ACUM_FOR_FMA
204
+#undef MMHA_USE_FP32_ACUM_FOR_OUT
csrc/cache_kernels.cu
+5
-5
@@ -48,7 +48,7 @@ __global__ void reshape_and_cache_kernel(
48
const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
49
const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
50
scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
51
- scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, block_size, head_size]
51
+ scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
52
const int* __restrict__ slot_mapping, // [num_tokens]
53
const int num_heads,
54
const int head_size,
@@ -73,10 +73,10 @@ __global__ void reshape_and_cache_kernel(
73
+ x_idx * block_size * x
74
+ block_offset * x
75
+ x_offset;
76
- const int tgt_value_idx = block_idx * num_heads * block_size * head_size
77
- + head_idx * block_size * head_size
78
- + block_offset * head_size
79
- + head_offset;
76
+ const int tgt_value_idx = block_idx * num_heads * head_size * block_size
77
+ + head_idx * head_size * block_size
78
+ + head_offset * block_size
79
+ + block_offset;
80
key_cache[tgt_key_idx] = __ldg(&key[src_idx]);
81
value_cache[tgt_value_idx] = __ldg(&value[src_idx]);
82
}
csrc/cuda_primitives.h
new
+1318
@@ -0,0 +1,1318 @@
1
+#pragma once
2
+
3
+#include <stdint.h>
4
+
5
+namespace cacheflow {
6
+////////////////////////////////////////////////////////////////////////////////////////////////////
7
+
8
+struct Float8_ {
9
+ float2 x;
10
+ float2 y;
11
+ float2 z;
12
+ float2 w;
13
+};
14
+
15
+////////////////////////////////////////////////////////////////////////////////////////////////////
16
+
17
+struct Float4_ {
18
+ float2 x;
19
+ float2 y;
20
+};
21
+
22
+////////////////////////////////////////////////////////////////////////////////////////////////////
23
+
24
+#ifdef ENABLE_BF16
25
+struct bf16_4_t {
26
+ __nv_bfloat162 x;
27
+ __nv_bfloat162 y;
28
+};
29
+
30
+////////////////////////////////////////////////////////////////////////////////////////////////////
31
+
32
+struct bf16_8_t {
33
+ __nv_bfloat162 x;
34
+ __nv_bfloat162 y;
35
+ __nv_bfloat162 z;
36
+ __nv_bfloat162 w;
37
+};
38
+#endif
39
+
40
+////////////////////////////////////////////////////////////////////////////////////////////////////
41
+
42
+inline __device__ float add(float a, float b)
43
+{
44
+ return a + b;
45
+}
46
+
47
+////////////////////////////////////////////////////////////////////////////////////////////////////
48
+
49
+inline __device__ float2 add(float2 a, float2 b)
50
+{
51
+ float2 c;
52
+ c.x = add(a.x, b.x);
53
+ c.y = add(a.y, b.y);
54
+ return c;
55
+}
56
+
57
+////////////////////////////////////////////////////////////////////////////////////////////////////
58
+
59
+inline __device__ float4 add(float4 a, float4 b)
60
+{
61
+ float4 c;
62
+ c.x = add(a.x, b.x);
63
+ c.y = add(a.y, b.y);
64
+ c.z = add(a.z, b.z);
65
+ c.w = add(a.w, b.w);
66
+ return c;
67
+}
68
+
69
+////////////////////////////////////////////////////////////////////////////////////////////////////
70
+
71
+#ifdef ENABLE_BF16
72
+inline __device__ __nv_bfloat16 add(__nv_bfloat16 a, __nv_bfloat16 b)
73
+{
74
+ return a + b;
75
+}
76
+
77
+////////////////////////////////////////////////////////////////////////////////////////////////////
78
+
79
+inline __device__ __nv_bfloat162 add(__nv_bfloat162 a, __nv_bfloat162 b)
80
+{
81
+ return bf16hadd2(a, b);
82
+}
83
+
84
+////////////////////////////////////////////////////////////////////////////////////////////////////
85
+
86
+inline __device__ bf16_4_t add(bf16_4_t a, bf16_4_t b)
87
+{
88
+ bf16_4_t c;
89
+ c.x = add(a.x, b.x);
90
+ c.y = add(a.y, b.y);
91
+ return c;
92
+}
93
+
94
+////////////////////////////////////////////////////////////////////////////////////////////////////
95
+
96
+inline __device__ bf16_8_t add(bf16_8_t a, bf16_8_t b)
97
+{
98
+ bf16_8_t c;
99
+ c.x = add(a.x, b.x);
100
+ c.y = add(a.y, b.y);
101
+ c.z = add(a.z, b.z);
102
+ c.w = add(a.w, b.w);
103
+ return c;
104
+}
105
+#endif // ENABLE_BF16
106
+
107
+////////////////////////////////////////////////////////////////////////////////////////////////////
108
+
109
+inline __device__ uint16_t add(uint16_t a, uint16_t b)
110
+{
111
+ uint16_t c;
112
+ asm volatile("add.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
113
+ return c;
114
+}
115
+
116
+////////////////////////////////////////////////////////////////////////////////////////////////////
117
+
118
+inline __device__ uint32_t add(uint32_t a, uint32_t b)
119
+{
120
+ uint32_t c;
121
+ asm volatile("add.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
122
+ return c;
123
+}
124
+
125
+////////////////////////////////////////////////////////////////////////////////////////////////////
126
+
127
+inline __device__ uint2 add(uint2 a, uint2 b)
128
+{
129
+ uint2 c;
130
+ c.x = add(a.x, b.x);
131
+ c.y = add(a.y, b.y);
132
+ return c;
133
+}
134
+
135
+////////////////////////////////////////////////////////////////////////////////////////////////////
136
+
137
+inline __device__ uint4 add(uint4 a, uint4 b)
138
+{
139
+ uint4 c;
140
+ c.x = add(a.x, b.x);
141
+ c.y = add(a.y, b.y);
142
+ c.z = add(a.z, b.z);
143
+ c.w = add(a.w, b.w);
144
+ return c;
145
+}
146
+
147
+////////////////////////////////////////////////////////////////////////////////////////////////////
148
+
149
+inline __device__ uint16_t float_to_half(float f)
150
+{
151
+ union {
152
+ uint32_t u32;
153
+ uint16_t u16[2];
154
+ } tmp;
155
+#if 0 && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 // Is it better?
156
+ float zero = 0.f;
157
+ asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(zero), "f"(f));
158
+#else
159
+ asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f));
160
+#endif
161
+ return tmp.u16[0];
162
+}
163
+
164
+////////////////////////////////////////////////////////////////////////////////////////////////////
165
+
166
+inline __device__ uint32_t float2_to_half2(float2 f)
167
+{
168
+ union {
169
+ uint32_t u32;
170
+ uint16_t u16[2];
171
+ } tmp;
172
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
173
+ asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(f.y), "f"(f.x));
174
+#else
175
+ asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
176
+ asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
177
+#endif
178
+ return tmp.u32;
179
+}
180
+
181
+////////////////////////////////////////////////////////////////////////////////////////////////////
182
+
183
+inline __device__ float half_to_float(uint16_t h)
184
+{
185
+ float f;
186
+ asm volatile("cvt.f32.f16 %0, %1;\n" : "=f"(f) : "h"(h));
187
+ return f;
188
+}
189
+
190
+////////////////////////////////////////////////////////////////////////////////////////////////////
191
+
192
+inline __device__ float2 half2_to_float2(uint32_t v)
193
+{
194
+ uint16_t lo, hi;
195
+ asm volatile("mov.b32 {%0, %1}, %2;\n" : "=h"(lo), "=h"(hi) : "r"(v));
196
+ return make_float2(half_to_float(lo), half_to_float(hi));
197
+}
198
+
199
+////////////////////////////////////////////////////////////////////////////////////////////////////
200
+
201
+inline __device__ float2 add(uint32_t a, float2 fb)
202
+{
203
+ float2 fa = half2_to_float2(a);
204
+ return add(fa, fb);
205
+}
206
+
207
+////////////////////////////////////////////////////////////////////////////////////////////////////
208
+
209
+inline __device__ Float4_ add(uint2 a, Float4_ fb)
210
+{
211
+ Float4_ fc;
212
+ fc.x = add(a.x, fb.x);
213
+ fc.y = add(a.y, fb.y);
214
+ return fc;
215
+}
216
+
217
+////////////////////////////////////////////////////////////////////////////////////////////////////
218
+
219
+inline __device__ Float8_ add(uint4 a, Float8_ fb)
220
+{
221
+ Float8_ fc;
222
+ fc.x = add(a.x, fb.x);
223
+ fc.y = add(a.y, fb.y);
224
+ fc.z = add(a.z, fb.z);
225
+ fc.w = add(a.w, fb.w);
226
+ return fc;
227
+}
228
+
229
+////////////////////////////////////////////////////////////////////////////////////////////////////
230
+
231
+inline __device__ uint32_t h0_h0(uint16_t a)
232
+{
233
+ uint32_t b;
234
+ asm volatile("mov.b32 %0, {%1, %1};" : "=r"(b) : "h"(a));
235
+ return b;
236
+}
237
+
238
+////////////////////////////////////////////////////////////////////////////////////////////////////
239
+
240
+inline __device__ float fma(float a, float b, float c)
241
+{
242
+ return a * b + c;
243
+}
244
+
245
+////////////////////////////////////////////////////////////////////////////////////////////////////
246
+
247
+inline __device__ float2 fma(float2 a, float2 b, float2 c)
248
+{
249
+ float2 d;
250
+ d.x = fma(a.x, b.x, c.x);
251
+ d.y = fma(a.y, b.y, c.y);
252
+ return d;
253
+}
254
+
255
+////////////////////////////////////////////////////////////////////////////////////////////////////
256
+
257
+inline __device__ float2 fma(float a, float2 b, float2 c)
258
+{
259
+ float2 d;
260
+ d.x = fma(a, b.x, c.x);
261
+ d.y = fma(a, b.y, c.y);
262
+ return d;
263
+}
264
+
265
+////////////////////////////////////////////////////////////////////////////////////////////////////
266
+
267
+inline __device__ float4 fma(float4 a, float4 b, float4 c)
268
+{
269
+ float4 d;
270
+ d.x = fma(a.x, b.x, c.x);
271
+ d.y = fma(a.y, b.y, c.y);
272
+ d.z = fma(a.z, b.z, c.z);
273
+ d.w = fma(a.w, b.w, c.w);
274
+ return d;
275
+}
276
+
277
+////////////////////////////////////////////////////////////////////////////////////////////////////
278
+
279
+inline __device__ float4 fma(float a, float4 b, float4 c)
280
+{
281
+ float4 d;
282
+ d.x = fma(a, b.x, c.x);
283
+ d.y = fma(a, b.y, c.y);
284
+ d.z = fma(a, b.z, c.z);
285
+ d.w = fma(a, b.w, c.w);
286
+ return d;
287
+}
288
+
289
+////////////////////////////////////////////////////////////////////////////////////////////////////
290
+
291
+inline __device__ Float4_ fma(float a, Float4_ b, Float4_ c)
292
+{
293
+ Float4_ d;
294
+ d.x = fma(a, b.x, c.x);
295
+ d.y = fma(a, b.y, c.y);
296
+ return d;
297
+}
298
+
299
+////////////////////////////////////////////////////////////////////////////////////////////////////
300
+
301
+inline __device__ Float8_ fma(float a, Float8_ b, Float8_ c)
302
+{
303
+ Float8_ d;
304
+ d.x = fma(a, b.x, c.x);
305
+ d.y = fma(a, b.y, c.y);
306
+ d.z = fma(a, b.z, c.z);
307
+ d.w = fma(a, b.w, c.w);
308
+ return d;
309
+}
310
+
311
+////////////////////////////////////////////////////////////////////////////////////////////////////
312
+
313
+#ifdef ENABLE_BF16
314
+inline __device__ float2 add(__nv_bfloat162 a, float2 fb)
315
+{
316
+ float2 fa = bf1622float2(a);
317
+ return add(fa, fb);
318
+}
319
+
320
+////////////////////////////////////////////////////////////////////////////////////////////////////
321
+
322
+inline __device__ Float4_ add(bf16_4_t a, Float4_ fb)
323
+{
324
+ Float4_ fc;
325
+ fc.x = add(a.x, fb.x);
326
+ fc.y = add(a.y, fb.y);
327
+ return fc;
328
+}
329
+
330
+////////////////////////////////////////////////////////////////////////////////////////////////////
331
+
332
+inline __device__ Float8_ add(bf16_8_t a, Float8_ fb)
333
+{
334
+ Float8_ fc;
335
+ fc.x = add(a.x, fb.x);
336
+ fc.y = add(a.y, fb.y);
337
+ fc.z = add(a.z, fb.z);
338
+ fc.w = add(a.w, fb.w);
339
+ return fc;
340
+}
341
+#endif // ENABLE_BF16
342
+
343
+////////////////////////////////////////////////////////////////////////////////////////////////////
344
+
345
+inline __device__ uint32_t fma(uint32_t a, uint32_t b, uint32_t c)
346
+{
347
+ uint32_t d;
348
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(d) : "r"(a), "r"(b), "r"(c));
349
+ return d;
350
+}
351
+
352
+////////////////////////////////////////////////////////////////////////////////////////////////////
353
+
354
+inline __device__ uint32_t fma(uint16_t a, uint32_t b, uint32_t c)
355
+{
356
+ return fma(h0_h0(a), b, c);
357
+}
358
+
359
+////////////////////////////////////////////////////////////////////////////////////////////////////
360
+
361
+inline __device__ uint2 fma(uint2 a, uint2 b, uint2 c)
362
+{
363
+ uint2 d;
364
+ d.x = fma(a.x, b.x, c.x);
365
+ d.y = fma(a.y, b.y, c.y);
366
+ return d;
367
+}
368
+
369
+////////////////////////////////////////////////////////////////////////////////////////////////////
370
+
371
+inline __device__ uint2 fma(uint16_t a, uint2 b, uint2 c)
372
+{
373
+ uint32_t s = h0_h0(a);
374
+ uint2 d;
375
+ d.x = fma(s, b.x, c.x);
376
+ d.y = fma(s, b.y, c.y);
377
+ return d;
378
+}
379
+
380
+////////////////////////////////////////////////////////////////////////////////////////////////////
381
+
382
+inline __device__ uint4 fma(uint4 a, uint4 b, uint4 c)
383
+{
384
+ uint4 d;
385
+ d.x = fma(a.x, b.x, c.x);
386
+ d.y = fma(a.y, b.y, c.y);
387
+ d.z = fma(a.z, b.z, c.z);
388
+ d.w = fma(a.w, b.w, c.w);
389
+ return d;
390
+}
391
+
392
+////////////////////////////////////////////////////////////////////////////////////////////////////
393
+
394
+inline __device__ uint4 fma(uint16_t a, uint4 b, uint4 c)
395
+{
396
+ uint32_t s = h0_h0(a);
397
+ uint4 d;
398
+ d.x = fma(s, b.x, c.x);
399
+ d.y = fma(s, b.y, c.y);
400
+ d.z = fma(s, b.z, c.z);
401
+ d.w = fma(s, b.w, c.w);
402
+ return d;
403
+}
404
+
405
+////////////////////////////////////////////////////////////////////////////////////////////////////
406
+
407
+inline __device__ float fma(uint16_t a, uint16_t b, float fc)
408
+{
409
+ float fa = half_to_float(a);
410
+ float fb = half_to_float(b);
411
+ return fa * fb + fc;
412
+}
413
+
414
+////////////////////////////////////////////////////////////////////////////////////////////////////
415
+
416
+inline __device__ float2 fma(uint32_t a, uint32_t b, float2 fc)
417
+{
418
+ float2 fa = half2_to_float2(a);
419
+ float2 fb = half2_to_float2(b);
420
+ return fma(fa, fb, fc);
421
+}
422
+
423
+////////////////////////////////////////////////////////////////////////////////////////////////////
424
+
425
+inline __device__ float2 fma(uint16_t a, uint32_t b, float2 fc)
426
+{
427
+ return fma(h0_h0(a), b, fc);
428
+}
429
+
430
+////////////////////////////////////////////////////////////////////////////////////////////////////
431
+
432
+inline __device__ Float4_ fma(uint2 a, uint2 b, Float4_ fc)
433
+{
434
+ Float4_ fd;
435
+ fd.x = fma(a.x, b.x, fc.x);
436
+ fd.y = fma(a.y, b.y, fc.y);
437
+ return fd;
438
+}
439
+
440
+////////////////////////////////////////////////////////////////////////////////////////////////////
441
+
442
+inline __device__ Float4_ fma(uint16_t a, uint2 b, Float4_ fc)
443
+{
444
+ uint32_t s = h0_h0(a);
445
+ Float4_ fd;
446
+ fd.x = fma(s, b.x, fc.x);
447
+ fd.y = fma(s, b.y, fc.y);
448
+ return fd;
449
+}
450
+
451
+////////////////////////////////////////////////////////////////////////////////////////////////////
452
+
453
+inline __device__ Float8_ fma(uint4 a, uint4 b, Float8_ fc)
454
+{
455
+ Float8_ fd;
456
+ fd.x = fma(a.x, b.x, fc.x);
457
+ fd.y = fma(a.y, b.y, fc.y);
458
+ fd.z = fma(a.z, b.z, fc.z);
459
+ fd.w = fma(a.w, b.w, fc.w);
460
+ return fd;
461
+}
462
+
463
+////////////////////////////////////////////////////////////////////////////////////////////////////
464
+
465
+inline __device__ Float8_ fma(uint16_t a, uint4 b, Float8_ fc)
466
+{
467
+ uint32_t s = h0_h0(a);
468
+ Float8_ fd;
469
+ fd.x = fma(s, b.x, fc.x);
470
+ fd.y = fma(s, b.y, fc.y);
471
+ fd.z = fma(s, b.z, fc.z);
472
+ fd.w = fma(s, b.w, fc.w);
473
+ return fd;
474
+}
475
+
476
+////////////////////////////////////////////////////////////////////////////////////////////////////
477
+#ifdef ENABLE_BF16
478
+inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b, __nv_bfloat162 c)
479
+{
480
+ return bf16hfma2(a, b, c);
481
+}
482
+
483
+////////////////////////////////////////////////////////////////////////////////////////////////////
484
+
485
+inline __device__ __nv_bfloat162 fma(__nv_bfloat16 a, __nv_bfloat162 b, __nv_bfloat162 c)
486
+{
487
+ return bf16hfma2(bf162bf162(a), b, c);
488
+}
489
+////////////////////////////////////////////////////////////////////////////////////////////////////
490
+
491
+inline __device__ bf16_4_t fma(bf16_4_t a, bf16_4_t b, bf16_4_t c)
492
+{
493
+ bf16_4_t d;
494
+ d.x = fma(a.x, b.x, c.x);
495
+ d.y = fma(a.y, b.y, c.y);
496
+ return d;
497
+}
498
+
499
+////////////////////////////////////////////////////////////////////////////////////////////////////
500
+
501
+inline __device__ bf16_4_t fma(__nv_bfloat16 a, bf16_4_t b, bf16_4_t c)
502
+{
503
+ __nv_bfloat162 s = bf162bf162(a);
504
+ bf16_4_t d;
505
+ d.x = fma(s, b.x, c.x);
506
+ d.y = fma(s, b.y, c.y);
507
+ return d;
508
+}
509
+
510
+////////////////////////////////////////////////////////////////////////////////////////////////////
511
+
512
+inline __device__ bf16_8_t fma(bf16_8_t a, bf16_8_t b, bf16_8_t c)
513
+{
514
+ bf16_8_t d;
515
+ d.x = fma(a.x, b.x, c.x);
516
+ d.y = fma(a.y, b.y, c.y);
517
+ d.z = fma(a.z, b.z, c.z);
518
+ d.w = fma(a.w, b.w, c.w);
519
+ return d;
520
+}
521
+
522
+////////////////////////////////////////////////////////////////////////////////////////////////////
523
+
524
+inline __device__ bf16_8_t fma(__nv_bfloat16 a, bf16_8_t b, bf16_8_t c)
525
+{
526
+ __nv_bfloat162 s = bf162bf162(a);
527
+ bf16_8_t d;
528
+ d.x = fma(s, b.x, c.x);
529
+ d.y = fma(s, b.y, c.y);
530
+ d.z = fma(s, b.z, c.z);
531
+ d.w = fma(s, b.w, c.w);
532
+ return d;
533
+}
534
+
535
+////////////////////////////////////////////////////////////////////////////////////////////////////
536
+
537
+inline __device__ float fma(__nv_bfloat16 a, __nv_bfloat16 b, float fc)
538
+{
539
+ return __bfloat162float(a) * __bfloat162float(b) + fc;
540
+}
541
+
542
+////////////////////////////////////////////////////////////////////////////////////////////////////
543
+
544
+inline __device__ float2 fma(__nv_bfloat162 a, __nv_bfloat162 b, float2 fc)
545
+{
546
+ float2 fa = bf1622float2(a);
547
+ float2 fb = bf1622float2(b);
548
+ return fma(fa, fb, fc);
549
+}
550
+
551
+////////////////////////////////////////////////////////////////////////////////////////////////////
552
+
553
+inline __device__ float2 fma(__nv_bfloat16 a, __nv_bfloat162 b, float2 fc)
554
+{
555
+ return fma(bf162bf162(a), b, fc);
556
+}
557
+
558
+////////////////////////////////////////////////////////////////////////////////////////////////////
559
+
560
+inline __device__ Float4_ fma(bf16_4_t a, bf16_4_t b, Float4_ fc)
561
+{
562
+ Float4_ fd;
563
+ fd.x = fma(a.x, b.x, fc.x);
564
+ fd.y = fma(a.y, b.y, fc.y);
565
+ return fd;
566
+}
567
+
568
+////////////////////////////////////////////////////////////////////////////////////////////////////
569
+
570
+inline __device__ Float4_ fma(__nv_bfloat16 a, bf16_4_t b, Float4_ fc)
571
+{
572
+ __nv_bfloat162 s = bf162bf162(a);
573
+ Float4_ fd;
574
+ fd.x = fma(s, b.x, fc.x);
575
+ fd.y = fma(s, b.y, fc.y);
576
+ return fd;
577
+}
578
+
579
+////////////////////////////////////////////////////////////////////////////////////////////////////
580
+
581
+inline __device__ Float8_ fma(bf16_8_t a, bf16_8_t b, Float8_ fc)
582
+{
583
+ Float8_ fd;
584
+ fd.x = fma(a.x, b.x, fc.x);
585
+ fd.y = fma(a.y, b.y, fc.y);
586
+ fd.z = fma(a.z, b.z, fc.z);
587
+ fd.w = fma(a.w, b.w, fc.w);
588
+ return fd;
589
+}
590
+
591
+////////////////////////////////////////////////////////////////////////////////////////////////////
592
+
593
+inline __device__ Float8_ fma(__nv_bfloat16 a, bf16_8_t b, Float8_ fc)
594
+{
595
+ __nv_bfloat162 s = bf162bf162(a);
596
+ Float8_ fd;
597
+ fd.x = fma(s, b.x, fc.x);
598
+ fd.y = fma(s, b.y, fc.y);
599
+ fd.z = fma(s, b.z, fc.z);
600
+ fd.w = fma(s, b.w, fc.w);
601
+ return fd;
602
+}
603
+#endif // ENABLE_BF16
604
+////////////////////////////////////////////////////////////////////////////////////////////////////
605
+
606
+template<typename Acc, typename A, typename B>
607
+inline __device__ Acc mul(A a, B b);
608
+
609
+////////////////////////////////////////////////////////////////////////////////////////////////////
610
+
611
+template<>
612
+inline __device__ float mul<float, float>(float a, float b)
613
+{
614
+ return a * b;
615
+}
616
+
617
+////////////////////////////////////////////////////////////////////////////////////////////////////
618
+
619
+template<>
620
+inline __device__ float2 mul(float2 a, float2 b)
621
+{
622
+ float2 c;
623
+ c.x = a.x * b.x;
624
+ c.y = a.y * b.y;
625
+ return c;
626
+}
627
+
628
+////////////////////////////////////////////////////////////////////////////////////////////////////
629
+
630
+template<>
631
+inline __device__ float2 mul(float a, float2 b)
632
+{
633
+ float2 c;
634
+ c.x = a * b.x;
635
+ c.y = a * b.y;
636
+ return c;
637
+}
638
+
639
+////////////////////////////////////////////////////////////////////////////////////////////////////
640
+
641
+template<>
642
+inline __device__ float4 mul(float4 a, float4 b)
643
+{
644
+ float4 c;
645
+ c.x = a.x * b.x;
646
+ c.y = a.y * b.y;
647
+ c.z = a.z * b.z;
648
+ c.w = a.w * b.w;
649
+ return c;
650
+}
651
+
652
+////////////////////////////////////////////////////////////////////////////////////////////////////
653
+
654
+template<>
655
+inline __device__ float4 mul(float a, float4 b)
656
+{
657
+ float4 c;
658
+ c.x = a * b.x;
659
+ c.y = a * b.y;
660
+ c.z = a * b.z;
661
+ c.w = a * b.w;
662
+ return c;
663
+}
664
+
665
+////////////////////////////////////////////////////////////////////////////////////////////////////
666
+
667
+template<>
668
+inline __device__ uint16_t mul(uint16_t a, uint16_t b)
669
+{
670
+ uint16_t c;
671
+ asm volatile("mul.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
672
+ return c;
673
+}
674
+
675
+////////////////////////////////////////////////////////////////////////////////////////////////////
676
+
677
+template<>
678
+inline __device__ uint32_t mul(uint32_t a, uint32_t b)
679
+{
680
+ uint32_t c;
681
+ asm volatile("mul.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
682
+ return c;
683
+}
684
+
685
+////////////////////////////////////////////////////////////////////////////////////////////////////
686
+
687
+template<>
688
+inline __device__ uint32_t mul(uint16_t a, uint32_t b)
689
+{
690
+ return mul<uint32_t, uint32_t, uint32_t>(h0_h0(a), b);
691
+}
692
+
693
+////////////////////////////////////////////////////////////////////////////////////////////////////
694
+
695
+template<>
696
+inline __device__ uint2 mul(uint2 a, uint2 b)
697
+{
698
+ uint2 c;
699
+ c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
700
+ c.y = mul<uint32_t, uint32_t, uint32_t>(a.y, b.y);
701
+ return c;
702
+}
703
+
704
+////////////////////////////////////////////////////////////////////////////////////////////////////
705
+
706
+template<>
707
+inline __device__ uint2 mul(uint16_t a, uint2 b)
708
+{
709
+ uint32_t s = h0_h0(a);
710
+ uint2 c;
711
+ c.x = mul<uint32_t, uint32_t, uint32_t>(s, b.x);
712
+ c.y = mul<uint32_t, uint32_t, uint32_t>(s, b.y);
713
+ return c;
714
+}
715
+
716
+////////////////////////////////////////////////////////////////////////////////////////////////////
717
+
718
+template<>
719
+inline __device__ uint4 mul(uint4 a, uint4 b)
720
+{
721
+ uint4 c;
722
+ c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
723
+ c.y = mul<uint32_t, uint32_t, uint32_t>(a.y, b.y);
724
+ c.z = mul<uint32_t, uint32_t, uint32_t>(a.z, b.z);
725
+ c.w = mul<uint32_t, uint32_t, uint32_t>(a.w, b.w);
726
+ return c;
727
+}
728
+
729
+////////////////////////////////////////////////////////////////////////////////////////////////////
730
+
731
+template<>
732
+inline __device__ uint4 mul(uint16_t a, uint4 b)
733
+{
734
+ uint32_t s = h0_h0(a);
735
+ uint4 c;
736
+ c.x = mul<uint32_t, uint32_t, uint32_t>(s, b.x);
737
+ c.y = mul<uint32_t, uint32_t, uint32_t>(s, b.y);
738
+ c.z = mul<uint32_t, uint32_t, uint32_t>(s, b.z);
739
+ c.w = mul<uint32_t, uint32_t, uint32_t>(s, b.w);
740
+ return c;
741
+}
742
+
743
+////////////////////////////////////////////////////////////////////////////////////////////////////
744
+
745
+template<>
746
+inline __device__ float mul(uint16_t a, uint16_t b)
747
+{
748
+ float fa = half_to_float(a);
749
+ float fb = half_to_float(b);
750
+ return fa * fb;
751
+}
752
+
753
+////////////////////////////////////////////////////////////////////////////////////////////////////
754
+
755
+template<>
756
+inline __device__ float2 mul(uint32_t a, uint32_t b)
757
+{
758
+ float2 fa = half2_to_float2(a);
759
+ float2 fb = half2_to_float2(b);
760
+ return mul<float2, float2, float2>(fa, fb);
761
+}
762
+
763
+////////////////////////////////////////////////////////////////////////////////////////////////////
764
+
765
+template<>
766
+inline __device__ float2 mul(uint16_t a, uint32_t b)
767
+{
768
+ return mul<float2, uint32_t, uint32_t>(h0_h0(a), b);
769
+}
770
+
771
+////////////////////////////////////////////////////////////////////////////////////////////////////
772
+
773
+template<>
774
+inline __device__ Float4_ mul(uint2 a, uint2 b)
775
+{
776
+ Float4_ fc;
777
+ fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
778
+ fc.y = mul<float2, uint32_t, uint32_t>(a.y, b.y);
779
+ return fc;
780
+}
781
+
782
+////////////////////////////////////////////////////////////////////////////////////////////////////
783
+
784
+template<>
785
+inline __device__ Float4_ mul(uint16_t a, uint2 b)
786
+{
787
+ uint32_t s = h0_h0(a);
788
+ Float4_ fc;
789
+ fc.x = mul<float2, uint32_t, uint32_t>(s, b.x);
790
+ fc.y = mul<float2, uint32_t, uint32_t>(s, b.y);
791
+ return fc;
792
+}
793
+
794
+////////////////////////////////////////////////////////////////////////////////////////////////////
795
+
796
+template<>
797
+inline __device__ Float8_ mul(uint4 a, uint4 b)
798
+{
799
+ Float8_ fc;
800
+ fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
801
+ fc.y = mul<float2, uint32_t, uint32_t>(a.y, b.y);
802
+ fc.z = mul<float2, uint32_t, uint32_t>(a.z, b.z);
803
+ fc.w = mul<float2, uint32_t, uint32_t>(a.w, b.w);
804
+ return fc;
805
+}
806
+
807
+////////////////////////////////////////////////////////////////////////////////////////////////////
808
+
809
+template<>
810
+inline __device__ Float8_ mul(uint16_t a, uint4 b)
811
+{
812
+ uint32_t s = h0_h0(a);
813
+ Float8_ fc;
814
+ fc.x = mul<float2, uint32_t, uint32_t>(s, b.x);
815
+ fc.y = mul<float2, uint32_t, uint32_t>(s, b.y);
816
+ fc.z = mul<float2, uint32_t, uint32_t>(s, b.z);
817
+ fc.w = mul<float2, uint32_t, uint32_t>(s, b.w);
818
+ return fc;
819
+}
820
+
821
+////////////////////////////////////////////////////////////////////////////////////////////////////
822
+
823
+#ifdef ENABLE_BF16
824
+template<>
825
+inline __device__ __nv_bfloat16 mul(__nv_bfloat16 a, __nv_bfloat16 b)
826
+{
827
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
828
+ return __hmul(a, b);
829
+#else
830
+ return bf16hmul(a, b);
831
+#endif
832
+}
833
+
834
+////////////////////////////////////////////////////////////////////////////////////////////////////
835
+
836
+template<>
837
+inline __device__ __nv_bfloat162 mul(__nv_bfloat162 a, __nv_bfloat162 b)
838
+{
839
+ return bf16hmul2(a, b);
840
+}
841
+
842
+////////////////////////////////////////////////////////////////////////////////////////////////////
843
+
844
+template<>
845
+inline __device__ __nv_bfloat162 mul(__nv_bfloat16 a, __nv_bfloat162 b)
846
+{
847
+ return mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
848
+}
849
+
850
+////////////////////////////////////////////////////////////////////////////////////////////////////
851
+
852
+template<>
853
+inline __device__ bf16_4_t mul(bf16_4_t a, bf16_4_t b)
854
+{
855
+ bf16_4_t c;
856
+ c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
857
+ c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
858
+ return c;
859
+}
860
+
861
+////////////////////////////////////////////////////////////////////////////////////////////////////
862
+
863
+template<>
864
+inline __device__ bf16_4_t mul(__nv_bfloat16 a, bf16_4_t b)
865
+{
866
+ __nv_bfloat162 s = bf162bf162(a);
867
+ bf16_4_t c;
868
+ c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.x);
869
+ c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.y);
870
+ return c;
871
+}
872
+
873
+////////////////////////////////////////////////////////////////////////////////////////////////////
874
+
875
+template<>
876
+inline __device__ bf16_8_t mul(bf16_8_t a, bf16_8_t b)
877
+{
878
+ bf16_8_t c;
879
+ c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
880
+ c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
881
+ c.z = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.z, b.z);
882
+ c.w = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.w, b.w);
883
+ return c;
884
+}
885
+
886
+////////////////////////////////////////////////////////////////////////////////////////////////////
887
+
888
+template<>
889
+inline __device__ bf16_8_t mul(__nv_bfloat16 a, bf16_8_t b)
890
+{
891
+ __nv_bfloat162 s = bf162bf162(a);
892
+ bf16_8_t c;
893
+ c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.x);
894
+ c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.y);
895
+ c.z = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.z);
896
+ c.w = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.w);
897
+ return c;
898
+}
899
+
900
+////////////////////////////////////////////////////////////////////////////////////////////////////
901
+
902
+template<>
903
+inline __device__ float mul(__nv_bfloat16 a, __nv_bfloat16 b)
904
+{
905
+ float fa = (float)a;
906
+ float fb = (float)b;
907
+ return fa * fb;
908
+}
909
+
910
+////////////////////////////////////////////////////////////////////////////////////////////////////
911
+
912
+template<>
913
+inline __device__ float2 mul(__nv_bfloat162 a, __nv_bfloat162 b)
914
+{
915
+ float2 fa = bf1622float2(a);
916
+ float2 fb = bf1622float2(b);
917
+ return mul<float2, float2, float2>(fa, fb);
918
+}
919
+
920
+////////////////////////////////////////////////////////////////////////////////////////////////////
921
+
922
+template<>
923
+inline __device__ float2 mul(__nv_bfloat16 a, __nv_bfloat162 b)
924
+{
925
+ return mul<float2, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
926
+}
927
+
928
+////////////////////////////////////////////////////////////////////////////////////////////////////
929
+
930
+template<>
931
+inline __device__ Float4_ mul(bf16_4_t a, bf16_4_t b)
932
+{
933
+ Float4_ fc;
934
+ fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
935
+ fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
936
+ return fc;
937
+}
938
+
939
+////////////////////////////////////////////////////////////////////////////////////////////////////
940
+
941
+template<>
942
+inline __device__ Float4_ mul(__nv_bfloat16 a, bf16_4_t b)
943
+{
944
+ __nv_bfloat162 s = bf162bf162(a);
945
+ Float4_ fc;
946
+ fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.x);
947
+ fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.y);
948
+ return fc;
949
+}
950
+
951
+////////////////////////////////////////////////////////////////////////////////////////////////////
952
+
953
+template<>
954
+inline __device__ Float8_ mul(bf16_8_t a, bf16_8_t b)
955
+{
956
+ Float8_ fc;
957
+ fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
958
+ fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
959
+ fc.z = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.z, b.z);
960
+ fc.w = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.w, b.w);
961
+ return fc;
962
+}
963
+
964
+////////////////////////////////////////////////////////////////////////////////////////////////////
965
+
966
+template<>
967
+inline __device__ Float8_ mul(__nv_bfloat16 a, bf16_8_t b)
968
+{
969
+ __nv_bfloat162 s = bf162bf162(a);
970
+ Float8_ fc;
971
+ fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.x);
972
+ fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.y);
973
+ fc.z = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.z);
974
+ fc.w = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.w);
975
+ return fc;
976
+}
977
+#endif // ENABLE_BF16
978
+////////////////////////////////////////////////////////////////////////////////////////////////////
979
+
980
+inline __device__ float sum(float v)
981
+{
982
+ return v;
983
+}
984
+
985
+////////////////////////////////////////////////////////////////////////////////////////////////////
986
+
987
+inline __device__ float sum(float2 v)
988
+{
989
+ return v.x + v.y;
990
+}
991
+
992
+////////////////////////////////////////////////////////////////////////////////////////////////////
993
+
994
+inline __device__ float sum(float4 v)
995
+{
996
+ return v.x + v.y + v.z + v.w;
997
+}
998
+
999
+////////////////////////////////////////////////////////////////////////////////////////////////////
1000
+
1001
+#ifdef ENABLE_BF16
1002
+inline __device__ float sum(__nv_bfloat162 v)
1003
+{
1004
+ float2 vf = bf1622float2(v);
1005
+ return vf.x + vf.y;
1006
+}
1007
+
1008
+////////////////////////////////////////////////////////////////////////////////////////////////////
1009
+
1010
+inline __device__ float sum(bf16_4_t v)
1011
+{
1012
+ return sum(v.x) + sum(v.y);
1013
+}
1014
+
1015
+////////////////////////////////////////////////////////////////////////////////////////////////////
1016
+
1017
+inline __device__ float sum(bf16_8_t v)
1018
+{
1019
+ return sum(v.x) + sum(v.y) + sum(v.z) + sum(v.w);
1020
+}
1021
+#endif // ENABLE_BF16
1022
+////////////////////////////////////////////////////////////////////////////////////////////////////
1023
+
1024
+inline __device__ float sum(uint16_t v)
1025
+{
1026
+ return half_to_float(v);
1027
+}
1028
+
1029
+////////////////////////////////////////////////////////////////////////////////////////////////////
1030
+
1031
+inline __device__ float sum(uint32_t v)
1032
+{
1033
+ float2 tmp = half2_to_float2(v);
1034
+ return tmp.x + tmp.y;
1035
+}
1036
+
1037
+////////////////////////////////////////////////////////////////////////////////////////////////////
1038
+
1039
+inline __device__ float sum(uint2 v)
1040
+{
1041
+ uint32_t c = add(v.x, v.y);
1042
+ return sum(c);
1043
+}
1044
+
1045
+////////////////////////////////////////////////////////////////////////////////////////////////////
1046
+
1047
+inline __device__ float sum(uint4 v)
1048
+{
1049
+#if 1
1050
+ uint32_t c = add(v.x, v.y);
1051
+ c = add(c, v.z);
1052
+ c = add(c, v.w);
1053
+#else
1054
+ uint32_t c = add(v.x, v.y);
1055
+ uint32_t d = add(v.z, v.w);
1056
+ c = add(c, d);
1057
+#endif
1058
+ return sum(c);
1059
+}
1060
+
1061
+////////////////////////////////////////////////////////////////////////////////////////////////////
1062
+
1063
+inline __device__ float sum(Float4_ v)
1064
+{
1065
+ return v.x.x + v.x.y + v.y.x + v.y.y;
1066
+}
1067
+
1068
+////////////////////////////////////////////////////////////////////////////////////////////////////
1069
+
1070
+inline __device__ float sum(Float8_ v)
1071
+{
1072
+ return v.x.x + v.x.y + v.y.x + v.y.y + v.z.x + v.z.y + v.w.x + v.w.y;
1073
+}
1074
+
1075
+////////////////////////////////////////////////////////////////////////////////////////////////////
1076
+
1077
+inline __device__ float dot(Float4_ a, Float4_ b)
1078
+{
1079
+ float2 acc = mul<float2, float2, float2>(a.x, b.x);
1080
+ acc = fma(a.y, b.y, acc);
1081
+ return acc.x + acc.y;
1082
+}
1083
+
1084
+////////////////////////////////////////////////////////////////////////////////////////////////////
1085
+
1086
+inline __device__ float dot(Float8_ a, Float8_ b)
1087
+{
1088
+ float2 acc = mul<float2, float2, float2>(a.x, b.x);
1089
+ acc = fma(a.y, b.y, acc);
1090
+ acc = fma(a.z, b.z, acc);
1091
+ acc = fma(a.w, b.w, acc);
1092
+ return acc.x + acc.y;
1093
+}
1094
+
1095
+////////////////////////////////////////////////////////////////////////////////////////////////////
1096
+
1097
+template<typename T>
1098
+inline __device__ float dot(T a, T b)
1099
+{
1100
+ return sum(mul<T, T, T>(a, b));
1101
+}
1102
+
1103
+////////////////////////////////////////////////////////////////////////////////////////////////////
1104
+
1105
+template<typename A, typename T>
1106
+inline __device__ float dot(T a, T b)
1107
+{
1108
+ return sum(mul<A, T, T>(a, b));
1109
+}
1110
+
1111
+////////////////////////////////////////////////////////////////////////////////////////////////////
1112
+
1113
+inline __device__ void zero(uint16_t& dst)
1114
+{
1115
+ dst = uint16_t(0);
1116
+}
1117
+
1118
+////////////////////////////////////////////////////////////////////////////////////////////////////
1119
+
1120
+template<typename T>
1121
+inline __device__ void zero(T& dst)
1122
+{
1123
+ constexpr int WORDS = sizeof(T) / 4;
1124
+ union {
1125
+ T raw;
1126
+ uint32_t words[WORDS];
1127
+ } tmp;
1128
+#pragma unroll
1129
+ for (int ii = 0; ii < WORDS; ++ii) {
1130
+ tmp.words[ii] = 0u;
1131
+ }
1132
+ dst = tmp.raw;
1133
+}
1134
+
1135
+
1136
+////////////////////////////////////////////////////////////////////////////////////////////////////
1137
+
1138
+inline __device__ void convert_from_float(float& dst, float src)
1139
+{
1140
+ dst = src;
1141
+}
1142
+
1143
+////////////////////////////////////////////////////////////////////////////////////////////////////
1144
+
1145
+inline __device__ void convert_from_float(uint16_t& dst, float src)
1146
+{
1147
+ dst = float_to_half(src);
1148
+}
1149
+
1150
+////////////////////////////////////////////////////////////////////////////////////////////////////
1151
+
1152
+inline __device__ void convert_from_float(uint32_t& dst, float2 src)
1153
+{
1154
+ dst = float2_to_half2(src);
1155
+}
1156
+
1157
+////////////////////////////////////////////////////////////////////////////////////////////////////
1158
+#ifdef ENABLE_BF16
1159
+inline __device__ void convert_from_float(__nv_bfloat16& dst, float src)
1160
+{
1161
+ dst = __float2bfloat16(src);
1162
+}
1163
+
1164
+////////////////////////////////////////////////////////////////////////////////////////////////////
1165
+
1166
+inline __device__ void convert_from_float(__nv_bfloat162& dst, float2 src)
1167
+{
1168
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1169
+ dst = __float22bfloat162_rn(src);
1170
+#else
1171
+ dst = __floats2bfloat162_rn(src.x, src.y);
1172
+#endif
1173
+}
1174
+#endif // ENABLE_BF16
1175
+////////////////////////////////////////////////////////////////////////////////////////////////////
1176
+
1177
+inline __device__ void convert_from_float(uint2& dst, Float4_ src)
1178
+{
1179
+ dst.x = float2_to_half2(src.x);
1180
+ dst.y = float2_to_half2(src.y);
1181
+}
1182
+
1183
+////////////////////////////////////////////////////////////////////////////////////////////////////
1184
+
1185
+inline __device__ void convert_from_float(uint4& dst, Float8_ src)
1186
+{
1187
+ dst.x = float2_to_half2(src.x);
1188
+ dst.y = float2_to_half2(src.y);
1189
+ dst.z = float2_to_half2(src.z);
1190
+ dst.w = float2_to_half2(src.w);
1191
+}
1192
+
1193
+////////////////////////////////////////////////////////////////////////////////////////////////////
1194
+
1195
+#ifdef ENABLE_BF16
1196
+inline __device__ void convert_from_float(bf16_4_t& dst, Float4_ src)
1197
+{
1198
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1199
+ dst.x = __float22bfloat162_rn(src.x);
1200
+ dst.y = __float22bfloat162_rn(src.y);
1201
+#else
1202
+ dst.x = __floats2bfloat162_rn(src.x.x, src.x.y);
1203
+ dst.y = __floats2bfloat162_rn(src.y.x, src.y.y);
1204
+#endif
1205
+}
1206
+
1207
+////////////////////////////////////////////////////////////////////////////////////////////////////
1208
+
1209
+inline __device__ void convert_from_float(bf16_8_t& dst, Float8_ src)
1210
+{
1211
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1212
+ dst.x = __float22bfloat162_rn(src.x);
1213
+ dst.y = __float22bfloat162_rn(src.y);
1214
+ dst.z = __float22bfloat162_rn(src.z);
1215
+ dst.w = __float22bfloat162_rn(src.w);
1216
+#else
1217
+ dst.x = __floats2bfloat162_rn(src.x.x, src.x.y);
1218
+ dst.y = __floats2bfloat162_rn(src.y.x, src.y.y);
1219
+ dst.z = __floats2bfloat162_rn(src.z.x, src.z.y);
1220
+ dst.w = __floats2bfloat162_rn(src.w.x, src.w.y);
1221
+#endif
1222
+}
1223
+#endif // ENABLE_BF16
1224
+
1225
+////////////////////////////////////////////////////////////////////////////////////////////////////
1226
+
1227
+inline __device__ void convert_from_float(float2& dst, float2 src)
1228
+{
1229
+ dst = src;
1230
+}
1231
+
1232
+////////////////////////////////////////////////////////////////////////////////////////////////////
1233
+
1234
+inline __device__ void convert_from_float(float4& dst, float4 src)
1235
+{
1236
+ dst = src;
1237
+}
1238
+
1239
+////////////////////////////////////////////////////////////////////////////////////////////////////
1240
+
1241
+inline __device__ float convert_to_float(float4 u)
1242
+{
1243
+ return u.x;
1244
+}
1245
+
1246
+////////////////////////////////////////////////////////////////////////////////////////////////////
1247
+
1248
+inline __device__ float convert_to_float(uint4 u)
1249
+{
1250
+ float2 tmp = half2_to_float2(u.x);
1251
+ return tmp.x;
1252
+}
1253
+
1254
+////////////////////////////////////////////////////////////////////////////////////////////////////
1255
+
1256
+inline __device__ float cast_to_float(float u)
1257
+{
1258
+ return u;
1259
+}
1260
+
1261
+////////////////////////////////////////////////////////////////////////////////////////////////////
1262
+
1263
+inline __device__ float2 cast_to_float(float2 u)
1264
+{
1265
+ return u;
1266
+}
1267
+
1268
+////////////////////////////////////////////////////////////////////////////////////////////////////
1269
+
1270
+inline __device__ float4 cast_to_float(float4 u)
1271
+{
1272
+ return u;
1273
+}
1274
+
1275
+////////////////////////////////////////////////////////////////////////////////////////////////////
1276
+
1277
+inline __device__ Float4_ cast_to_float(Float4_ u)
1278
+{
1279
+ return u;
1280
+}
1281
+
1282
+////////////////////////////////////////////////////////////////////////////////////////////////////
1283
+
1284
+inline __device__ Float8_ cast_to_float(Float8_ u)
1285
+{
1286
+ return u;
1287
+}
1288
+
1289
+////////////////////////////////////////////////////////////////////////////////////////////////////
1290
+
1291
+inline __device__ float2 cast_to_float(uint32_t u)
1292
+{
1293
+ return half2_to_float2(u);
1294
+}
1295
+
1296
+////////////////////////////////////////////////////////////////////////////////////////////////////
1297
+
1298
+inline __device__ Float4_ cast_to_float(uint2 u)
1299
+{
1300
+ Float4_ tmp;
1301
+ tmp.x = half2_to_float2(u.x);
1302
+ tmp.y = half2_to_float2(u.y);
1303
+ return tmp;
1304
+}
1305
+
1306
+////////////////////////////////////////////////////////////////////////////////////////////////////
1307
+
1308
+inline __device__ Float8_ cast_to_float(uint4 u)
1309
+{
1310
+ Float8_ tmp;
1311
+ tmp.x = half2_to_float2(u.x);
1312
+ tmp.y = half2_to_float2(u.y);
1313
+ tmp.z = half2_to_float2(u.z);
1314
+ tmp.w = half2_to_float2(u.w);
1315
+ return tmp;
1316
+}
1317
+
1318
+}
setup.py
+9
-2
@@ -9,15 +9,22 @@ ext_modules = []
9
10
# Cache operations.
11
cache_extension = cpp_extension.CUDAExtension(
12
- name='cacheflow.ops',
12
+ name='cacheflow.cache_ops',
13
sources=['csrc/cache.cpp', 'csrc/cache_kernels.cu'],
14
extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
15
)
16
ext_modules.append(cache_extension)
17
18
+# Attention kernels.
19
+attention_extension = cpp_extension.CUDAExtension(
20
+ name='cacheflow.attention_ops',
21
+ sources=['csrc/attention.cpp', 'csrc/attention_kernels.cu'],
22
+ extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
23
+)
24
+ext_modules.append(attention_extension)
25
+
26
setuptools.setup(
27
name='cacheflow',
20
- requires_python='>=3.9',
28
ext_modules=ext_modules,
29
cmdclass={'build_ext': cpp_extension.BuildExtension},
30
)
tests/kernels/attention.py
new
+142
@@ -0,0 +1,142 @@
1
+import random
2
+from typing import Optional
3
+
4
+import torch
5
+
6
+from cacheflow import attention_ops
7
+
8
+
9
+def ref_masked_attention(
10
+ query: torch.Tensor,
11
+ key: torch.Tensor,
12
+ value: torch.Tensor,
13
+ scale: float,
14
+ attn_mask: Optional[torch.Tensor] = None,
15
+) -> torch.Tensor:
16
+ query = query * scale
17
+ attn = torch.einsum('qhd,khd->hqk', query, key)
18
+ if attn_mask is not None:
19
+ attn = attn + attn_mask
20
+ attn = torch.softmax(attn, dim=-1)
21
+ out = torch.einsum('hqk,khd->qhd', attn, value)
22
+ return out
23
+
24
+
25
+def ref_single_query_cached_kv_attention(
26
+ output: torch.Tensor,
27
+ query: torch.Tensor,
28
+ key_cache: torch.Tensor,
29
+ value_cache: torch.Tensor,
30
+ block_tables: torch.Tensor,
31
+ context_lens: torch.Tensor,
32
+) -> None:
33
+ num_heads = value_cache.shape[1]
34
+ head_size = value_cache.shape[2]
35
+ block_size = value_cache.shape[3]
36
+
37
+ num_input_tokens = query.shape[0]
38
+ for i in range(num_input_tokens):
39
+ q = query[i].unsqueeze(0)
40
+ block_table = block_tables[i]
41
+ context_len = int(context_lens[i])
42
+
43
+ keys = []
44
+ values = []
45
+ for j in range(context_len):
46
+ block_number = int(block_table[j // block_size])
47
+ block_offset = j % block_size
48
+
49
+ k = key_cache[block_number, :, :, block_offset, :]
50
+ k = k.reshape(num_heads, head_size)
51
+ keys.append(k)
52
+
53
+ v = value_cache[block_number, :, :, block_offset]
54
+ values.append(v)
55
+ keys = torch.stack(keys, dim=0)
56
+ values = torch.stack(values, dim=0)
57
+
58
+ scale = 1.0 / (head_size ** 0.5)
59
+ out = ref_masked_attention(q, keys, values, scale)
60
+ out = out.view(num_heads, head_size)
61
+ output[i].copy_(out, non_blocking=True)
62
+
63
+
64
+def test_single_query_cached_kv_attention(
65
+ num_tokens: int,
66
+ num_heads: int,
67
+ head_size: int,
68
+ block_size: int,
69
+ num_blocks: int,
70
+ dtype: torch.dtype,
71
+) -> None:
72
+ query = torch.randn(
73
+ num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
74
+ x = 16 // torch.tensor([], dtype=dtype).element_size()
75
+ key_block_shape = (num_heads, head_size // x, block_size, x)
76
+ key_cache = torch.randn(
77
+ size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
78
+ value_block_shape = (num_heads, head_size, block_size)
79
+ value_cache = torch.randn(
80
+ size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
81
+
82
+ context_lens = [random.randint(1, 4096) for _ in range(num_tokens)]
83
+ max_context_len = max(context_lens)
84
+ context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
85
+
86
+ max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
87
+ block_tables = []
88
+ for _ in range(num_tokens):
89
+ block_table = [
90
+ random.randint(0, num_blocks - 1)
91
+ for _ in range(max_num_blocks_per_seq)
92
+ ]
93
+ block_tables.append(block_table)
94
+ block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
95
+
96
+ scale = float(1.0 / (head_size ** 0.5))
97
+ output = torch.empty_like(query)
98
+ attention_ops.single_query_cached_kv_attention(
99
+ output,
100
+ query,
101
+ key_cache,
102
+ value_cache,
103
+ scale,
104
+ block_tables,
105
+ context_lens,
106
+ block_size,
107
+ max_context_len,
108
+ )
109
+
110
+ ref_output = torch.empty_like(query)
111
+ ref_single_query_cached_kv_attention(
112
+ ref_output,
113
+ query,
114
+ key_cache,
115
+ value_cache,
116
+ block_tables,
117
+ context_lens,
118
+ )
119
+ # NOTE(woosuk): Due to the difference in the data types the two
120
+ # implementations use for attention softmax logits and accumulation,
121
+ # there is a small difference in the final outputs.
122
+ # We should use a relaxed tolerance for the test.
123
+ assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
124
+
125
+
126
+@torch.inference_mode()
127
+def test_attention() -> None:
128
+ for dtype in [torch.half, torch.float]:
129
+ for block_size in [8, 16]:
130
+ for head_size in [64, 80, 96, 128, 256]:
131
+ test_single_query_cached_kv_attention(
132
+ num_tokens=37,
133
+ num_heads=3,
134
+ head_size=head_size,
135
+ block_size=block_size,
136
+ num_blocks=1024,
137
+ dtype=dtype,
138
+ )
139
+
140
+
141
+if __name__ == '__main__':
142
+ test_attention()
tests/kernels/cache.py
renamed
+8
-8
@@ -2,7 +2,7 @@ import random
2
3
import torch
4
5
-from cacheflow.ops import reshape_and_cache
5
+from cacheflow import cache_ops
6
7
8
def test_reshape_and_cache(
@@ -26,30 +26,30 @@ def test_reshape_and_cache(
26
key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
27
cloned_key_cache = key_cache.clone()
28
29
- value_cache_shape = (num_blocks, num_heads, block_size, head_size)
29
+ value_cache_shape = (num_blocks, num_heads, head_size, block_size)
30
value_cache = torch.randn(
31
size=value_cache_shape, dtype=dtype, device='cuda')
32
cloned_value_cache = value_cache.clone()
33
34
- reshape_and_cache(key, value, key_cache, value_cache, slot_mapping)
34
+ cache_ops.reshape_and_cache(key, value, key_cache, value_cache, slot_mapping)
35
36
for i in range(num_tokens):
37
reshaped_key = key.reshape(num_tokens, num_heads, head_size // x, x)
38
block_idx = slot_mapping[i] // block_size
39
block_offset = slot_mapping[i] % block_size
40
cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
41
- cloned_value_cache[block_idx, :, block_offset, :] = value[i]
41
+ cloned_value_cache[block_idx, :, :, block_offset] = value[i]
42
43
assert torch.allclose(key_cache, cloned_key_cache)
44
assert torch.allclose(value_cache, cloned_value_cache)
45
46
47
-@torch.no_grad()
48
-def test_kernels():
47
+@torch.inference_mode()
48
+def test_cache() -> None:
49
test_reshape_and_cache(
50
- num_tokens=3, num_heads=2, head_size=16, block_size=2, num_blocks=2,
50
+ num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
51
dtype=torch.half)
52
53
54
if __name__ == '__main__':
55
- test_kernels()
55
+ test_cache()