A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Refactor attention kernels (#53)
Woosuk Kwon committed
May 3, 2023 at 13:40 UTC
436e523bf1616bea22a1df78d93d7522311dccc8
14 files changed
+1253
-2569
csrc/attention/attention_dtypes.cuh
new
+5
@@ -0,0 +1,5 @@
1
+#pragma once
2
+
3
+#include "attention_generic.cuh"
4
+#include "dtype_float16.cuh"
5
+#include "dtype_float32.cuh"
csrc/attention/attention_generic.cuh
new
+47
@@ -0,0 +1,47 @@
1
+#pragma once
2
+
3
+#include <stdint.h>
4
+
5
+namespace cacheflow {
6
+
7
+// A vector type to store Q, K, V elements.
8
+template<typename T, int VEC_SIZE>
9
+struct Vec {};
10
+
11
+// A vector type to store FP32 accumulators.
12
+template<typename T>
13
+struct FloatVec {};
14
+
15
+// Template vector operations.
16
+template<typename Acc, typename A, typename B>
17
+inline __device__ Acc mul(A a, B b);
18
+
19
+template<typename T>
20
+inline __device__ float sum(T v);
21
+
22
+template<typename T>
23
+inline __device__ float dot(T a, T b) {
24
+ return sum(mul<T, T, T>(a, b));
25
+}
26
+
27
+template<typename A, typename T>
28
+inline __device__ float dot(T a, T b) {
29
+ return sum(mul<A, T, T>(a, b));
30
+}
31
+
32
+template<typename T>
33
+inline __device__ void zero(T& dst) {
34
+ constexpr int WORDS = sizeof(T) / 4;
35
+ union {
36
+ T raw;
37
+ uint32_t words[WORDS];
38
+ } tmp;
39
+
40
+#pragma unroll
41
+ for (int ii = 0; ii < WORDS; ++ii) {
42
+ tmp.words[ii] = 0u;
43
+ }
44
+ dst = tmp.raw;
45
+}
46
+
47
+} // namespace cacheflow
csrc/attention/attention_kernels.cu
new
+451
@@ -0,0 +1,451 @@
1
+#include <torch/extension.h>
2
+#include <ATen/cuda/CUDAContext.h>
3
+
4
+#include "attention_dtypes.cuh"
5
+#include "attention_utils.cuh"
6
+
7
+#include <algorithm>
8
+
9
+#define WARP_SIZE 32
10
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
11
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
12
+
13
+namespace cacheflow {
14
+
15
+// Utility function for attention softmax.
16
+template<int NUM_WARPS>
17
+inline __device__ float block_sum(float* red_smem, float sum) {
18
+ // Decompose the thread index into warp / lane.
19
+ int warp = threadIdx.x / WARP_SIZE;
20
+ int lane = threadIdx.x % WARP_SIZE;
21
+
22
+ // Compute the sum per warp.
23
+#pragma unroll
24
+ for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
25
+ sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
26
+ }
27
+
28
+ // Warp leaders store the data to shared memory.
29
+ if (lane == 0) {
30
+ red_smem[warp] = sum;
31
+ }
32
+
33
+ // Make sure the data is in shared memory.
34
+ __syncthreads();
35
+
36
+ // The warps compute the final sums.
37
+ if (lane < NUM_WARPS) {
38
+ sum = red_smem[lane];
39
+ }
40
+
41
+ // Parallel reduction inside the warp.
42
+#pragma unroll
43
+ for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
44
+ sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
45
+ }
46
+
47
+ // Broadcast to other threads.
48
+ return __shfl_sync(uint32_t(-1), sum, 0);
49
+}
50
+
51
+// Grid: (num_heads, num_seqs).
52
+template<
53
+ typename scalar_t,
54
+ int HEAD_SIZE,
55
+ int BLOCK_SIZE,
56
+ int NUM_THREADS>
57
+__global__ void single_query_cached_kv_attention_kernel(
58
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
59
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
60
+ const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
61
+ const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
62
+ const float scale,
63
+ const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
64
+ const int* __restrict__ context_lens, // [num_seqs]
65
+ const int max_num_blocks_per_seq,
66
+ const int q_stride) {
67
+ constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
68
+ constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
69
+ constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
70
+ const int thread_idx = threadIdx.x;
71
+ const int warp_idx = thread_idx / WARP_SIZE;
72
+ const int lane = thread_idx % WARP_SIZE;
73
+
74
+ const int head_idx = blockIdx.x;
75
+ const int num_heads = gridDim.x;
76
+ const int seq_idx = blockIdx.y;
77
+
78
+ // A vector type to store a part of a key or a query.
79
+ // The vector size is configured in such a way that the threads in a thread group
80
+ // fetch or compute 16 bytes at a time.
81
+ // For example, if the size of a thread group is 4 and the data type is half,
82
+ // then the vector size is 16 / (4 * sizeof(half)) == 2.
83
+ constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
84
+ using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
85
+ using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
86
+
87
+ constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
88
+ constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
89
+
90
+ const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
91
+ const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
92
+
93
+ // Load the query to registers.
94
+ // Each thread in a thread group has a different part of the query.
95
+ // For example, if the the thread group size is 4, then the first thread in the group
96
+ // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
97
+ // th vectors of the query, and so on.
98
+ // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
99
+ const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
100
+ Q_vec q_vecs[NUM_VECS_PER_THREAD];
101
+#pragma unroll
102
+ for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
103
+ const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
104
+ q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
105
+ }
106
+
107
+ // Memory planning.
108
+ extern __shared__ char shared_mem[];
109
+ // NOTE(woosuk): We use FP32 for the softmax logits for better accuracy.
110
+ float* logits = reinterpret_cast<float*>(shared_mem);
111
+ // Workspace for reduction.
112
+ __shared__ float red_smem[2 * NUM_WARPS];
113
+
114
+ // x == THREAD_GROUP_SIZE * VEC_SIZE
115
+ // Each thread group fetches x elements from the key at a time.
116
+ constexpr int x = 16 / sizeof(scalar_t);
117
+ float qk_max = -FLT_MAX;
118
+
119
+ const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
120
+ const int context_len = context_lens[seq_idx];
121
+ const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
122
+
123
+ // Iterate over the key blocks.
124
+ // Each warp fetches a block of keys for each iteration.
125
+ // Each thread group in a warp fetches a key from the block, and computes
126
+ // dot product with the query.
127
+ for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
128
+ const int physical_block_number = block_table[block_idx];
129
+
130
+ // Load a key to registers.
131
+ // Each thread in a thread group has a different part of the key.
132
+ // For example, if the the thread group size is 4, then the first thread in the group
133
+ // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
134
+ // vectors of the key, and so on.
135
+ for (int i = 0; i < NUM_TOKENS_PER_THREAD_GROUP; i++) {
136
+ const int physical_block_offset = (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;
137
+ const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
138
+ K_vec k_vecs[NUM_VECS_PER_THREAD];
139
+
140
+#pragma unroll
141
+ for (int j = 0; j < NUM_VECS_PER_THREAD; j++) {
142
+ const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
143
+ + head_idx * HEAD_SIZE * BLOCK_SIZE
144
+ + physical_block_offset * x;
145
+ const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
146
+ const int offset1 = (vec_idx * VEC_SIZE) / x;
147
+ const int offset2 = (vec_idx * VEC_SIZE) % x;
148
+ k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
149
+ }
150
+
151
+ // Compute dot product.
152
+ // This includes a reduction across the threads in the same thread group.
153
+ const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
154
+ const bool mask = token_idx >= context_len;
155
+
156
+ if (thread_group_offset == 0) {
157
+ // Store the partial reductions to shared memory.
158
+ // NOTE(woosuk): It is required to zero out the masked logits.
159
+ logits[token_idx] = mask ? 0.f : qk;
160
+ // Update the max value.
161
+ qk_max = mask ? qk_max : fmaxf(qk_max, qk);
162
+ }
163
+ }
164
+ }
165
+
166
+ // Perform reduction across the threads in the same warp to get the
167
+ // max qk value for each "warp" (not across the thread block yet).
168
+ // The 0-th thread of each thread group already has its max qk value.
169
+#pragma unroll
170
+ for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
171
+ qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
172
+ }
173
+ if (lane == 0) {
174
+ red_smem[warp_idx] = qk_max;
175
+ }
176
+ __syncthreads();
177
+
178
+ // TODO(woosuk): Refactor this part.
179
+ // Get the max qk value for the sequence.
180
+ qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
181
+#pragma unroll
182
+ for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
183
+ qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
184
+ }
185
+ // Broadcast the max qk value to all threads.
186
+ qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
187
+
188
+ // Get the sum of the exp values.
189
+ float exp_sum = 0.f;
190
+ for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
191
+ float val = __expf(logits[i] - qk_max);
192
+ logits[i] = val;
193
+ exp_sum += val;
194
+ }
195
+ exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
196
+
197
+ // Compute softmax.
198
+ const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
199
+ for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
200
+ logits[i] *= inv_sum;
201
+ }
202
+ __syncthreads();
203
+
204
+ // Each thread will fetch 16 bytes from the value cache at a time.
205
+ constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
206
+ using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
207
+ using L_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
208
+ using Float_L_vec = typename FloatVec<L_vec>::Type;
209
+
210
+ constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
211
+ constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
212
+ constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
213
+
214
+ // NOTE(woosuk): We use FP32 for the accumulator for better accuracy.
215
+ float accs[NUM_ROWS_PER_THREAD];
216
+#pragma unroll
217
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
218
+ accs[i] = 0.f;
219
+ }
220
+
221
+ for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
222
+ const int physical_block_number = block_table[block_idx];
223
+ const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
224
+ const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
225
+ L_vec logits_vec;
226
+ from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx));
227
+
228
+ const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
229
+ + head_idx * HEAD_SIZE * BLOCK_SIZE;
230
+#pragma unroll
231
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
232
+ const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
233
+ if (row_idx < HEAD_SIZE) {
234
+ const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
235
+ V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
236
+ accs[i] += dot(logits_vec, v_vec);
237
+ }
238
+ }
239
+ }
240
+
241
+ // Perform reduction within each warp.
242
+#pragma unroll
243
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
244
+ float acc = accs[i];
245
+#pragma unroll
246
+ for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
247
+ acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
248
+ }
249
+ accs[i] = acc;
250
+ }
251
+
252
+ // NOTE(woosuk): A barrier is required because the shared memory space for logits
253
+ // is reused for the output.
254
+ __syncthreads();
255
+
256
+ // Perform reduction across warps.
257
+ float* out_smem = reinterpret_cast<float*>(shared_mem);
258
+#pragma unroll
259
+ for (int i = NUM_WARPS; i > 1; i /= 2) {
260
+ int mid = i / 2;
261
+ // Upper warps write to shared memory.
262
+ if (warp_idx >= mid && warp_idx < i) {
263
+ float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
264
+#pragma unroll
265
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
266
+ const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
267
+ if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
268
+ dst[row_idx] = accs[i];
269
+ }
270
+ }
271
+ }
272
+ __syncthreads();
273
+
274
+ // Lower warps update the output.
275
+ if (warp_idx < mid) {
276
+ const float* src = &out_smem[warp_idx * HEAD_SIZE];
277
+#pragma unroll
278
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
279
+ const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
280
+ if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
281
+ accs[i] += src[row_idx];
282
+ }
283
+ }
284
+ }
285
+ __syncthreads();
286
+ }
287
+
288
+ // Write the final output.
289
+ if (warp_idx == 0) {
290
+ scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
291
+#pragma unroll
292
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
293
+ const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
294
+ if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
295
+ from_float(*(out_ptr + row_idx), accs[i]);
296
+ }
297
+ }
298
+ }
299
+}
300
+
301
+} // namespace cacheflow
302
+
303
+#define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
304
+ cacheflow::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
305
+ <<<grid, block, shared_mem_size, stream>>>( \
306
+ out_ptr, \
307
+ query_ptr, \
308
+ key_cache_ptr, \
309
+ value_cache_ptr, \
310
+ scale, \
311
+ block_tables_ptr, \
312
+ context_lens_ptr, \
313
+ max_num_blocks_per_seq, \
314
+ query_stride);
315
+
316
+// TODO(woosuk): Tune NUM_THREADS.
317
+template<
318
+ typename T,
319
+ int BLOCK_SIZE,
320
+ int NUM_THREADS = 128>
321
+void single_query_cached_kv_attention_launcher(
322
+ torch::Tensor& out,
323
+ torch::Tensor& query,
324
+ torch::Tensor& key_cache,
325
+ torch::Tensor& value_cache,
326
+ float scale,
327
+ torch::Tensor& block_tables,
328
+ torch::Tensor& context_lens,
329
+ int max_context_len) {
330
+ int num_seqs = query.size(0);
331
+ int num_heads = query.size(1);
332
+ int head_size = query.size(2);
333
+ int max_num_blocks_per_seq = block_tables.size(1);
334
+ int query_stride = query.stride(0);
335
+
336
+ int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
337
+ assert(head_size % thread_group_size == 0);
338
+
339
+ T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
340
+ T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
341
+ T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
342
+ T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
343
+ int* block_tables_ptr = block_tables.data_ptr<int>();
344
+ int* context_lens_ptr = context_lens.data_ptr<int>();
345
+
346
+ constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
347
+ int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
348
+ int logits_size = padded_max_context_len * sizeof(T);
349
+ int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
350
+ int shared_mem_size = std::max(logits_size, outputs_size);
351
+
352
+ dim3 grid(num_heads, num_seqs);
353
+ dim3 block(NUM_THREADS);
354
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
355
+ switch (head_size) {
356
+ case 32:
357
+ LAUNCH_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
358
+ break;
359
+ case 64:
360
+ LAUNCH_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
361
+ break;
362
+ case 80:
363
+ LAUNCH_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
364
+ break;
365
+ case 96:
366
+ LAUNCH_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
367
+ break;
368
+ case 128:
369
+ LAUNCH_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
370
+ break;
371
+ case 160:
372
+ LAUNCH_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
373
+ break;
374
+ case 192:
375
+ LAUNCH_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
376
+ break;
377
+ case 256:
378
+ LAUNCH_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
379
+ break;
380
+ default:
381
+ TORCH_CHECK(false, "Unsupported head size: ", head_size);
382
+ break;
383
+ }
384
+}
385
+
386
+#define CALL_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
387
+ single_query_cached_kv_attention_launcher<T, BLOCK_SIZE>( \
388
+ out, \
389
+ query, \
390
+ key_cache, \
391
+ value_cache, \
392
+ scale, \
393
+ block_tables, \
394
+ context_lens, \
395
+ max_context_len);
396
+
397
+#define CALL_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
398
+ switch (block_size) { \
399
+ case 1: \
400
+ CALL_KERNEL_LAUNCHER(T, 1); \
401
+ break; \
402
+ case 2: \
403
+ CALL_KERNEL_LAUNCHER(T, 2); \
404
+ break; \
405
+ case 4: \
406
+ CALL_KERNEL_LAUNCHER(T, 4); \
407
+ break; \
408
+ case 8: \
409
+ CALL_KERNEL_LAUNCHER(T, 8); \
410
+ break; \
411
+ case 16: \
412
+ CALL_KERNEL_LAUNCHER(T, 16); \
413
+ break; \
414
+ case 32: \
415
+ CALL_KERNEL_LAUNCHER(T, 32); \
416
+ break; \
417
+ case 64: \
418
+ CALL_KERNEL_LAUNCHER(T, 64); \
419
+ break; \
420
+ case 128: \
421
+ CALL_KERNEL_LAUNCHER(T, 128); \
422
+ break; \
423
+ case 256: \
424
+ CALL_KERNEL_LAUNCHER(T, 256); \
425
+ break; \
426
+ default: \
427
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
428
+ break; \
429
+ }
430
+
431
+void single_query_cached_kv_attention(
432
+ torch::Tensor& out, // [num_seqs, num_heads, head_size]
433
+ torch::Tensor& query, // [num_seqs, num_heads, head_size]
434
+ torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
435
+ torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
436
+ float scale,
437
+ torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
438
+ torch::Tensor& context_lens, // [num_seqs]
439
+ int block_size,
440
+ int max_context_len) {
441
+ // TODO(woosuk): Support FP32 and BF16.
442
+ if (query.dtype() == at::ScalarType::Half) {
443
+ CALL_KERNEL_LAUNCHER_BLOCK_SIZE(uint16_t);
444
+ } else {
445
+ TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
446
+ }
447
+}
448
+
449
+#undef WARP_SIZE
450
+#undef MAX
451
+#undef MIN
csrc/attention/attention_utils.cuh
new
+38
@@ -0,0 +1,38 @@
1
+#pragma once
2
+
3
+#include "attention_dtypes.cuh"
4
+
5
+#include <float.h>
6
+#include <type_traits>
7
+
8
+namespace cacheflow {
9
+
10
+// Q*K^T operation.
11
+template<int THREAD_GROUP_SIZE, typename Vec, int N>
12
+inline __device__ float qk_dot_(const Vec (&q)[N], const Vec (&k)[N]) {
13
+ using A_vec = typename FloatVec<Vec>::Type;
14
+ // Compute the parallel products for Q*K^T (treat vector lanes separately).
15
+ A_vec qk_vec = mul<A_vec, Vec, Vec>(q[0], k[0]);
16
+#pragma unroll
17
+ for (int ii = 1; ii < N; ++ii) {
18
+ qk_vec = fma(q[ii], k[ii], qk_vec);
19
+ }
20
+
21
+ // Finalize the reduction across lanes.
22
+ float qk = sum(qk_vec);
23
+#pragma unroll
24
+ for (int mask = THREAD_GROUP_SIZE / 2; mask >= 1; mask /= 2) {
25
+ qk += __shfl_xor_sync(uint32_t(-1), qk, mask);
26
+ }
27
+ return qk;
28
+}
29
+
30
+template<typename T, int THREAD_GROUP_SIZE>
31
+struct Qk_dot {
32
+ template<typename Vec, int N>
33
+ static inline __device__ float dot(const Vec (&q)[N], const Vec (&k)[N]) {
34
+ return qk_dot_<THREAD_GROUP_SIZE>(q, k);
35
+ }
36
+};
37
+
38
+} // namespace cacheflow
csrc/attention/dtype_float16.cuh
new
+426
@@ -0,0 +1,426 @@
1
+#pragma once
2
+
3
+#include "attention_generic.cuh"
4
+#include "dtype_float32.cuh"
5
+
6
+#include <stdint.h>
7
+
8
+namespace cacheflow {
9
+
10
+// FP16 vector types for Q, K, V.
11
+template<>
12
+struct Vec<uint16_t, 1> {
13
+ using Type = uint16_t;
14
+};
15
+template<>
16
+struct Vec<uint16_t, 2> {
17
+ using Type = uint32_t;
18
+};
19
+template<>
20
+struct Vec<uint16_t, 4> {
21
+ using Type = uint2;
22
+};
23
+template<>
24
+struct Vec<uint16_t, 8> {
25
+ using Type = uint4;
26
+};
27
+
28
+// FP32 accumulator vector types corresponding to Vec.
29
+template<>
30
+struct FloatVec<uint16_t> {
31
+ using Type = float;
32
+};
33
+template<>
34
+struct FloatVec<uint32_t> {
35
+ using Type = float2;
36
+};
37
+template<>
38
+struct FloatVec<uint2> {
39
+ using Type = Float4_;
40
+};
41
+template<>
42
+struct FloatVec<uint4> {
43
+ using Type = Float8_;
44
+};
45
+
46
+// Utility functions for type conversions.
47
+inline __device__ uint32_t h0_h0(uint16_t a) {
48
+ uint32_t b;
49
+ asm volatile("mov.b32 %0, {%1, %1};" : "=r"(b) : "h"(a));
50
+ return b;
51
+}
52
+
53
+inline __device__ float half_to_float(uint16_t h) {
54
+ float f;
55
+ asm volatile("cvt.f32.f16 %0, %1;\n" : "=f"(f) : "h"(h));
56
+ return f;
57
+}
58
+
59
+inline __device__ float2 half2_to_float2(uint32_t v) {
60
+ uint16_t lo, hi;
61
+ asm volatile("mov.b32 {%0, %1}, %2;\n" : "=h"(lo), "=h"(hi) : "r"(v));
62
+ return make_float2(half_to_float(lo), half_to_float(hi));
63
+}
64
+
65
+inline __device__ uint16_t float_to_half(float f) {
66
+ union {
67
+ uint32_t u32;
68
+ uint16_t u16[2];
69
+ } tmp;
70
+ asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f));
71
+ return tmp.u16[0];
72
+}
73
+
74
+inline __device__ uint32_t float2_to_half2(float2 f) {
75
+ union {
76
+ uint32_t u32;
77
+ uint16_t u16[2];
78
+ } tmp;
79
+
80
+#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
81
+ asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(f.y), "f"(f.x));
82
+#else
83
+ asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
84
+ asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
85
+#endif
86
+ return tmp.u32;
87
+}
88
+
89
+// Vector addition.
90
+inline __device__ uint16_t add(uint16_t a, uint16_t b) {
91
+ uint16_t c;
92
+ asm volatile("add.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
93
+ return c;
94
+}
95
+
96
+inline __device__ uint32_t add(uint32_t a, uint32_t b) {
97
+ uint32_t c;
98
+ asm volatile("add.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
99
+ return c;
100
+}
101
+
102
+inline __device__ uint2 add(uint2 a, uint2 b) {
103
+ uint2 c;
104
+ c.x = add(a.x, b.x);
105
+ c.y = add(a.y, b.y);
106
+ return c;
107
+}
108
+
109
+inline __device__ uint4 add(uint4 a, uint4 b) {
110
+ uint4 c;
111
+ c.x = add(a.x, b.x);
112
+ c.y = add(a.y, b.y);
113
+ c.z = add(a.z, b.z);
114
+ c.w = add(a.w, b.w);
115
+ return c;
116
+}
117
+
118
+inline __device__ float2 add(uint32_t a, float2 fb) {
119
+ float2 fa = half2_to_float2(a);
120
+ return add(fa, fb);
121
+}
122
+
123
+inline __device__ Float4_ add(uint2 a, Float4_ fb) {
124
+ Float4_ fc;
125
+ fc.x = add(a.x, fb.x);
126
+ fc.y = add(a.y, fb.y);
127
+ return fc;
128
+}
129
+
130
+inline __device__ Float8_ add(uint4 a, Float8_ fb) {
131
+ Float8_ fc;
132
+ fc.x = add(a.x, fb.x);
133
+ fc.y = add(a.y, fb.y);
134
+ fc.z = add(a.z, fb.z);
135
+ fc.w = add(a.w, fb.w);
136
+ return fc;
137
+}
138
+
139
+// Vector multiplication.
140
+template<>
141
+inline __device__ uint16_t mul(uint16_t a, uint16_t b) {
142
+ uint16_t c;
143
+ asm volatile("mul.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
144
+ return c;
145
+}
146
+
147
+template<>
148
+inline __device__ uint32_t mul(uint32_t a, uint32_t b) {
149
+ uint32_t c;
150
+ asm volatile("mul.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
151
+ return c;
152
+}
153
+
154
+template<>
155
+inline __device__ uint32_t mul(uint16_t a, uint32_t b) {
156
+ return mul<uint32_t, uint32_t, uint32_t>(h0_h0(a), b);
157
+}
158
+
159
+template<>
160
+inline __device__ uint2 mul(uint2 a, uint2 b) {
161
+ uint2 c;
162
+ c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
163
+ c.y = mul<uint32_t, uint32_t, uint32_t>(a.y, b.y);
164
+ return c;
165
+}
166
+
167
+template<>
168
+inline __device__ uint2 mul(uint16_t a, uint2 b) {
169
+ uint32_t s = h0_h0(a);
170
+ uint2 c;
171
+ c.x = mul<uint32_t, uint32_t, uint32_t>(s, b.x);
172
+ c.y = mul<uint32_t, uint32_t, uint32_t>(s, b.y);
173
+ return c;
174
+}
175
+
176
+template<>
177
+inline __device__ uint4 mul(uint4 a, uint4 b) {
178
+ uint4 c;
179
+ c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
180
+ c.y = mul<uint32_t, uint32_t, uint32_t>(a.y, b.y);
181
+ c.z = mul<uint32_t, uint32_t, uint32_t>(a.z, b.z);
182
+ c.w = mul<uint32_t, uint32_t, uint32_t>(a.w, b.w);
183
+ return c;
184
+}
185
+
186
+template<>
187
+inline __device__ uint4 mul(uint16_t a, uint4 b) {
188
+ uint32_t s = h0_h0(a);
189
+ uint4 c;
190
+ c.x = mul<uint32_t, uint32_t, uint32_t>(s, b.x);
191
+ c.y = mul<uint32_t, uint32_t, uint32_t>(s, b.y);
192
+ c.z = mul<uint32_t, uint32_t, uint32_t>(s, b.z);
193
+ c.w = mul<uint32_t, uint32_t, uint32_t>(s, b.w);
194
+ return c;
195
+}
196
+
197
+template<>
198
+inline __device__ float mul(uint16_t a, uint16_t b) {
199
+ float fa = half_to_float(a);
200
+ float fb = half_to_float(b);
201
+ return fa * fb;
202
+}
203
+
204
+template<>
205
+inline __device__ float2 mul(uint32_t a, uint32_t b) {
206
+ float2 fa = half2_to_float2(a);
207
+ float2 fb = half2_to_float2(b);
208
+ return mul<float2, float2, float2>(fa, fb);
209
+}
210
+
211
+template<>
212
+inline __device__ float2 mul(uint16_t a, uint32_t b) {
213
+ return mul<float2, uint32_t, uint32_t>(h0_h0(a), b);
214
+}
215
+
216
+template<>
217
+inline __device__ Float4_ mul(uint2 a, uint2 b) {
218
+ Float4_ fc;
219
+ fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
220
+ fc.y = mul<float2, uint32_t, uint32_t>(a.y, b.y);
221
+ return fc;
222
+}
223
+
224
+template<>
225
+inline __device__ Float4_ mul(uint16_t a, uint2 b) {
226
+ uint32_t s = h0_h0(a);
227
+ Float4_ fc;
228
+ fc.x = mul<float2, uint32_t, uint32_t>(s, b.x);
229
+ fc.y = mul<float2, uint32_t, uint32_t>(s, b.y);
230
+ return fc;
231
+}
232
+
233
+template<>
234
+inline __device__ Float8_ mul(uint4 a, uint4 b) {
235
+ Float8_ fc;
236
+ fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
237
+ fc.y = mul<float2, uint32_t, uint32_t>(a.y, b.y);
238
+ fc.z = mul<float2, uint32_t, uint32_t>(a.z, b.z);
239
+ fc.w = mul<float2, uint32_t, uint32_t>(a.w, b.w);
240
+ return fc;
241
+}
242
+
243
+template<>
244
+inline __device__ Float8_ mul(uint16_t a, uint4 b) {
245
+ uint32_t s = h0_h0(a);
246
+ Float8_ fc;
247
+ fc.x = mul<float2, uint32_t, uint32_t>(s, b.x);
248
+ fc.y = mul<float2, uint32_t, uint32_t>(s, b.y);
249
+ fc.z = mul<float2, uint32_t, uint32_t>(s, b.z);
250
+ fc.w = mul<float2, uint32_t, uint32_t>(s, b.w);
251
+ return fc;
252
+}
253
+
254
+// Vector fused multiply-add.
255
+inline __device__ uint32_t fma(uint32_t a, uint32_t b, uint32_t c) {
256
+ uint32_t d;
257
+ asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(d) : "r"(a), "r"(b), "r"(c));
258
+ return d;
259
+}
260
+
261
+inline __device__ uint32_t fma(uint16_t a, uint32_t b, uint32_t c) {
262
+ return fma(h0_h0(a), b, c);
263
+}
264
+
265
+inline __device__ uint2 fma(uint2 a, uint2 b, uint2 c) {
266
+ uint2 d;
267
+ d.x = fma(a.x, b.x, c.x);
268
+ d.y = fma(a.y, b.y, c.y);
269
+ return d;
270
+}
271
+
272
+inline __device__ uint2 fma(uint16_t a, uint2 b, uint2 c) {
273
+ uint32_t s = h0_h0(a);
274
+ uint2 d;
275
+ d.x = fma(s, b.x, c.x);
276
+ d.y = fma(s, b.y, c.y);
277
+ return d;
278
+}
279
+
280
+inline __device__ uint4 fma(uint4 a, uint4 b, uint4 c) {
281
+ uint4 d;
282
+ d.x = fma(a.x, b.x, c.x);
283
+ d.y = fma(a.y, b.y, c.y);
284
+ d.z = fma(a.z, b.z, c.z);
285
+ d.w = fma(a.w, b.w, c.w);
286
+ return d;
287
+}
288
+
289
+inline __device__ uint4 fma(uint16_t a, uint4 b, uint4 c) {
290
+ uint32_t s = h0_h0(a);
291
+ uint4 d;
292
+ d.x = fma(s, b.x, c.x);
293
+ d.y = fma(s, b.y, c.y);
294
+ d.z = fma(s, b.z, c.z);
295
+ d.w = fma(s, b.w, c.w);
296
+ return d;
297
+}
298
+
299
+inline __device__ float fma(uint16_t a, uint16_t b, float fc) {
300
+ float fa = half_to_float(a);
301
+ float fb = half_to_float(b);
302
+ return fa * fb + fc;
303
+}
304
+
305
+inline __device__ float2 fma(uint32_t a, uint32_t b, float2 fc) {
306
+ float2 fa = half2_to_float2(a);
307
+ float2 fb = half2_to_float2(b);
308
+ return fma(fa, fb, fc);
309
+}
310
+
311
+inline __device__ float2 fma(uint16_t a, uint32_t b, float2 fc) {
312
+ return fma(h0_h0(a), b, fc);
313
+}
314
+
315
+inline __device__ Float4_ fma(uint2 a, uint2 b, Float4_ fc) {
316
+ Float4_ fd;
317
+ fd.x = fma(a.x, b.x, fc.x);
318
+ fd.y = fma(a.y, b.y, fc.y);
319
+ return fd;
320
+}
321
+
322
+inline __device__ Float4_ fma(uint16_t a, uint2 b, Float4_ fc) {
323
+ uint32_t s = h0_h0(a);
324
+ Float4_ fd;
325
+ fd.x = fma(s, b.x, fc.x);
326
+ fd.y = fma(s, b.y, fc.y);
327
+ return fd;
328
+}
329
+
330
+inline __device__ Float8_ fma(uint4 a, uint4 b, Float8_ fc) {
331
+ Float8_ fd;
332
+ fd.x = fma(a.x, b.x, fc.x);
333
+ fd.y = fma(a.y, b.y, fc.y);
334
+ fd.z = fma(a.z, b.z, fc.z);
335
+ fd.w = fma(a.w, b.w, fc.w);
336
+ return fd;
337
+}
338
+
339
+inline __device__ Float8_ fma(uint16_t a, uint4 b, Float8_ fc) {
340
+ uint32_t s = h0_h0(a);
341
+ Float8_ fd;
342
+ fd.x = fma(s, b.x, fc.x);
343
+ fd.y = fma(s, b.y, fc.y);
344
+ fd.z = fma(s, b.z, fc.z);
345
+ fd.w = fma(s, b.w, fc.w);
346
+ return fd;
347
+}
348
+
349
+// Vector sum.
350
+template<>
351
+inline __device__ float sum(uint16_t v) {
352
+ return half_to_float(v);
353
+}
354
+
355
+template<>
356
+inline __device__ float sum(uint32_t v) {
357
+ float2 tmp = half2_to_float2(v);
358
+ return tmp.x + tmp.y;
359
+}
360
+
361
+template<>
362
+inline __device__ float sum(uint2 v) {
363
+ uint32_t c = add(v.x, v.y);
364
+ return sum(c);
365
+}
366
+
367
+template<>
368
+inline __device__ float sum(uint4 v) {
369
+ uint32_t c = add(v.x, v.y);
370
+ c = add(c, v.z);
371
+ c = add(c, v.w);
372
+ return sum(c);
373
+}
374
+
375
+// Zero-out a vector.
376
+inline __device__ void zero(uint16_t& dst) {
377
+ dst = uint16_t(0);
378
+}
379
+
380
+// From float32 to float16.
381
+inline __device__ void from_float(uint16_t& dst, float src) {
382
+ dst = float_to_half(src);
383
+}
384
+
385
+inline __device__ void from_float(uint32_t& dst, float2 src) {
386
+ dst = float2_to_half2(src);
387
+}
388
+
389
+inline __device__ void from_float(uint2& dst, Float4_ src) {
390
+ dst.x = float2_to_half2(src.x);
391
+ dst.y = float2_to_half2(src.y);
392
+}
393
+
394
+inline __device__ void from_float(uint4& dst, Float8_ src) {
395
+ dst.x = float2_to_half2(src.x);
396
+ dst.y = float2_to_half2(src.y);
397
+ dst.z = float2_to_half2(src.z);
398
+ dst.w = float2_to_half2(src.w);
399
+}
400
+
401
+// From float16 to float32.
402
+inline __device__ float to_float(uint16_t u) {
403
+ return half_to_float(u);
404
+}
405
+
406
+inline __device__ float2 to_float(uint32_t u) {
407
+ return half2_to_float2(u);
408
+}
409
+
410
+inline __device__ Float4_ to_float(uint2 u) {
411
+ Float4_ tmp;
412
+ tmp.x = half2_to_float2(u.x);
413
+ tmp.y = half2_to_float2(u.y);
414
+ return tmp;
415
+}
416
+
417
+inline __device__ Float8_ to_float(uint4 u) {
418
+ Float8_ tmp;
419
+ tmp.x = half2_to_float2(u.x);
420
+ tmp.y = half2_to_float2(u.y);
421
+ tmp.z = half2_to_float2(u.z);
422
+ tmp.w = half2_to_float2(u.w);
423
+ return tmp;
424
+}
425
+
426
+} // namespace cacheflow
csrc/attention/dtype_float32.cuh
new
+250
@@ -0,0 +1,250 @@
1
+#pragma once
2
+
3
+#include "attention_generic.cuh"
4
+
5
+#include <stdint.h>
6
+
7
+namespace cacheflow {
8
+
9
+// Define FP32 vector data types.
10
+struct Float4_ {
11
+ float2 x;
12
+ float2 y;
13
+};
14
+
15
+struct Float8_ {
16
+ float2 x;
17
+ float2 y;
18
+ float2 z;
19
+ float2 w;
20
+};
21
+
22
+// FP32 vector types for Q, K, V.
23
+template<>
24
+struct Vec<float, 1> {
25
+ using Type = float;
26
+};
27
+template<>
28
+struct Vec<float, 2> {
29
+ using Type = float2;
30
+};
31
+template<>
32
+struct Vec<float, 4> {
33
+ using Type = float4;
34
+};
35
+
36
+// FP32 accumulator vector types corresponding to Vec.
37
+template<>
38
+struct FloatVec<float> {
39
+ using Type = float;
40
+};
41
+template<>
42
+struct FloatVec<float2> {
43
+ using Type = float2;
44
+};
45
+template<>
46
+struct FloatVec<float4> {
47
+ using Type = float4;
48
+};
49
+
50
+// Vector addition.
51
+inline __device__ float add(float a, float b) {
52
+ return a + b;
53
+}
54
+
55
+inline __device__ float2 add(float2 a, float2 b) {
56
+ float2 c;
57
+ c.x = add(a.x, b.x);
58
+ c.y = add(a.y, b.y);
59
+ return c;
60
+}
61
+
62
+inline __device__ float4 add(float4 a, float4 b) {
63
+ float4 c;
64
+ c.x = add(a.x, b.x);
65
+ c.y = add(a.y, b.y);
66
+ c.z = add(a.z, b.z);
67
+ c.w = add(a.w, b.w);
68
+ return c;
69
+}
70
+
71
+// Vector multiplication.
72
+template<>
73
+inline __device__ float mul<float, float>(float a, float b) {
74
+ return a * b;
75
+}
76
+
77
+template<>
78
+inline __device__ float2 mul(float2 a, float2 b) {
79
+ float2 c;
80
+ c.x = a.x * b.x;
81
+ c.y = a.y * b.y;
82
+ return c;
83
+}
84
+
85
+template<>
86
+inline __device__ float2 mul(float a, float2 b) {
87
+ float2 c;
88
+ c.x = a * b.x;
89
+ c.y = a * b.y;
90
+ return c;
91
+}
92
+
93
+template<>
94
+inline __device__ float4 mul(float4 a, float4 b) {
95
+ float4 c;
96
+ c.x = a.x * b.x;
97
+ c.y = a.y * b.y;
98
+ c.z = a.z * b.z;
99
+ c.w = a.w * b.w;
100
+ return c;
101
+}
102
+
103
+template<>
104
+inline __device__ float4 mul(float a, float4 b) {
105
+ float4 c;
106
+ c.x = a * b.x;
107
+ c.y = a * b.y;
108
+ c.z = a * b.z;
109
+ c.w = a * b.w;
110
+ return c;
111
+}
112
+
113
+// Vector fused multiply-add.
114
+inline __device__ float fma(float a, float b, float c) {
115
+ return a * b + c;
116
+}
117
+
118
+inline __device__ float2 fma(float2 a, float2 b, float2 c) {
119
+ float2 d;
120
+ d.x = fma(a.x, b.x, c.x);
121
+ d.y = fma(a.y, b.y, c.y);
122
+ return d;
123
+}
124
+
125
+inline __device__ float2 fma(float a, float2 b, float2 c) {
126
+ float2 d;
127
+ d.x = fma(a, b.x, c.x);
128
+ d.y = fma(a, b.y, c.y);
129
+ return d;
130
+}
131
+
132
+inline __device__ float4 fma(float4 a, float4 b, float4 c) {
133
+ float4 d;
134
+ d.x = fma(a.x, b.x, c.x);
135
+ d.y = fma(a.y, b.y, c.y);
136
+ d.z = fma(a.z, b.z, c.z);
137
+ d.w = fma(a.w, b.w, c.w);
138
+ return d;
139
+}
140
+
141
+inline __device__ float4 fma(float a, float4 b, float4 c) {
142
+ float4 d;
143
+ d.x = fma(a, b.x, c.x);
144
+ d.y = fma(a, b.y, c.y);
145
+ d.z = fma(a, b.z, c.z);
146
+ d.w = fma(a, b.w, c.w);
147
+ return d;
148
+}
149
+
150
+inline __device__ Float4_ fma(float a, Float4_ b, Float4_ c) {
151
+ Float4_ d;
152
+ d.x = fma(a, b.x, c.x);
153
+ d.y = fma(a, b.y, c.y);
154
+ return d;
155
+}
156
+
157
+inline __device__ Float8_ fma(float a, Float8_ b, Float8_ c) {
158
+ Float8_ d;
159
+ d.x = fma(a, b.x, c.x);
160
+ d.y = fma(a, b.y, c.y);
161
+ d.z = fma(a, b.z, c.z);
162
+ d.w = fma(a, b.w, c.w);
163
+ return d;
164
+}
165
+
166
+// Vector sum.
167
+template<>
168
+inline __device__ float sum(float v) {
169
+ return v;
170
+}
171
+
172
+template<>
173
+inline __device__ float sum(float2 v) {
174
+ return v.x + v.y;
175
+}
176
+
177
+template<>
178
+inline __device__ float sum(float4 v) {
179
+ return v.x + v.y + v.z + v.w;
180
+}
181
+
182
+template<>
183
+inline __device__ float sum(Float4_ v) {
184
+ return v.x.x + v.x.y + v.y.x + v.y.y;
185
+}
186
+
187
+template<>
188
+inline __device__ float sum(Float8_ v) {
189
+ return v.x.x + v.x.y + v.y.x + v.y.y + v.z.x + v.z.y + v.w.x + v.w.y;
190
+}
191
+
192
+// Vector dot product.
193
+inline __device__ float dot(float a, float b) {
194
+ return a * b;
195
+}
196
+
197
+inline __device__ float dot(float2 a, float2 b) {
198
+ float2 c = mul<float2, float2, float2>(a, b);
199
+ return c.x + c.y;
200
+}
201
+
202
+inline __device__ float dot(Float4_ a, Float4_ b) {
203
+ float2 acc = mul<float2, float2, float2>(a.x, b.x);
204
+ acc = fma(a.y, b.y, acc);
205
+ return acc.x + acc.y;
206
+}
207
+
208
+inline __device__ float dot(Float8_ a, Float8_ b) {
209
+ float2 acc = mul<float2, float2, float2>(a.x, b.x);
210
+ acc = fma(a.y, b.y, acc);
211
+ acc = fma(a.z, b.z, acc);
212
+ acc = fma(a.w, b.w, acc);
213
+ return acc.x + acc.y;
214
+}
215
+
216
+// From float to float.
217
+inline __device__ void from_float(float& dst, float src) {
218
+ dst = src;
219
+}
220
+
221
+inline __device__ void from_float(float2& dst, float2 src) {
222
+ dst = src;
223
+}
224
+
225
+inline __device__ void from_float(float4& dst, float4 src) {
226
+ dst = src;
227
+}
228
+
229
+// From float to float.
230
+inline __device__ float to_float(float u) {
231
+ return u;
232
+}
233
+
234
+inline __device__ float2 to_float(float2 u) {
235
+ return u;
236
+}
237
+
238
+inline __device__ float4 to_float(float4 u) {
239
+ return u;
240
+}
241
+
242
+inline __device__ Float4_ to_float(Float4_ u) {
243
+ return u;
244
+}
245
+
246
+inline __device__ Float8_ to_float(Float8_ u) {
247
+ return u;
248
+}
249
+
250
+} // namespace cacheflow
csrc/attention_kernels.cu
deleted
-896
@@ -1,896 +0,0 @@
1
-#include <torch/extension.h>
2
-#include <ATen/cuda/CUDAContext.h>
3
-
4
-#include "attention_utils.h"
5
-#include "cuda_primitives.h"
6
-#include "reduction_utils.h"
7
-
8
-#include <algorithm>
9
-
10
-#define WARP_SIZE 32
11
-#define MAX(a, b) ((a) > (b) ? (a) : (b))
12
-#define MIN(a, b) ((a) < (b) ? (a) : (b))
13
-
14
-namespace cacheflow {
15
-
16
-// Grid: (num_heads, num_seqs).
17
-template<
18
- typename scalar_t,
19
- int HEAD_SIZE,
20
- int BLOCK_SIZE,
21
- int NUM_THREADS>
22
-__global__ void single_query_cached_kv_attention_kernel(
23
- scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
24
- const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
25
- const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
26
- const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
27
- const float scale,
28
- const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
29
- const int* __restrict__ context_lens, // [num_seqs]
30
- const int max_num_blocks_per_seq,
31
- const int q_stride) {
32
- constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
33
- constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
34
- constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
35
- const int thread_idx = threadIdx.x;
36
- const int warp_idx = thread_idx / WARP_SIZE;
37
- const int lane = thread_idx % WARP_SIZE;
38
-
39
- const int head_idx = blockIdx.x;
40
- const int num_heads = gridDim.x;
41
- const int seq_idx = blockIdx.y;
42
-
43
- // A vector type to store a part of a key or a query.
44
- // The vector size is configured in such a way that the threads in a thread group
45
- // fetch or compute 16 bytes at a time.
46
- // For example, if the size of a thread group is 4 and the data type is half,
47
- // then the vector size is 16 / (4 * sizeof(half)) == 2.
48
- constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
49
- using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
50
- using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
51
-
52
- constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
53
- constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
54
-
55
- const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
56
- const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
57
-
58
- // Load the query to registers.
59
- // Each thread in a thread group has a different part of the query.
60
- // For example, if the the thread group size is 4, then the first thread in the group
61
- // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
62
- // th vectors of the query, and so on.
63
- // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
64
- const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
65
- Q_vec q_vecs[NUM_VECS_PER_THREAD];
66
-#pragma unroll
67
- for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
68
- const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
69
- q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
70
- }
71
-
72
- // Memory planning.
73
- extern __shared__ char shared_mem[];
74
- // NOTE(woosuk): We use FP32 logits and accumulation.
75
- float *logits = reinterpret_cast<float*>(shared_mem);
76
- // Workspace for reduction.
77
- __shared__ float red_smem[2 * NUM_WARPS];
78
-
79
- // x == THREAD_GROUP_SIZE * VEC_SIZE
80
- // Each thread group fetches x elements from the key at a time.
81
- constexpr int x = 16 / sizeof(scalar_t);
82
- float qk_max = -FLT_MAX;
83
-
84
- const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
85
- const int context_len = context_lens[seq_idx];
86
- const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
87
-
88
- // Iterate over the key blocks.
89
- // Each warp fetches a block of keys for each iteration.
90
- // Each thread group in a warp fetches a key from the block, and computes
91
- // dot product with the query.
92
- for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
93
- const int physical_block_number = block_table[block_idx];
94
-
95
- // Load a key to registers.
96
- // Each thread in a thread group has a different part of the key.
97
- // For example, if the the thread group size is 4, then the first thread in the group
98
- // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
99
- // vectors of the key, and so on.
100
- for (int i = 0; i < NUM_TOKENS_PER_THREAD_GROUP; i++) {
101
- const int physical_block_offset = (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;
102
- const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
103
- K_vec k_vecs[NUM_VECS_PER_THREAD];
104
-
105
-#pragma unroll
106
- for (int j = 0; j < NUM_VECS_PER_THREAD; j++) {
107
- const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
108
- + head_idx * HEAD_SIZE * BLOCK_SIZE
109
- + physical_block_offset * x;
110
- const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
111
- const int offset1 = (vec_idx * VEC_SIZE) / x;
112
- const int offset2 = (vec_idx * VEC_SIZE) % x;
113
- k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
114
- }
115
-
116
- // Compute dot product.
117
- // This includes a reduction across the threads in the same thread group.
118
- const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
119
- const bool mask = token_idx >= context_len;
120
-
121
- if (thread_group_offset == 0) {
122
- // Store the partial reductions to shared memory.
123
- // NOTE(woosuk): It is required to zero out the masked logits.
124
- logits[token_idx] = mask ? 0.f : qk;
125
- // Update the max value.
126
- qk_max = mask ? qk_max : fmaxf(qk_max, qk);
127
- }
128
- }
129
- }
130
-
131
- // Perform reduction across the threads in the same warp to get the
132
- // max qk value for each "warp" (not across the thread block yet).
133
- // The 0-th thread of each thread group already has its max qk value.
134
-#pragma unroll
135
- for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
136
- qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
137
- }
138
- if (lane == 0) {
139
- red_smem[warp_idx] = qk_max;
140
- }
141
- __syncthreads();
142
-
143
- // TODO(woosuk): Refactor this part.
144
- // Get the max qk value for the sequence.
145
- qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
146
-#pragma unroll
147
- for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
148
- qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
149
- }
150
- // Broadcast the max qk value to all threads.
151
- qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
152
-
153
- // Get the sum of the exp values.
154
- float exp_sum = 0.f;
155
- for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
156
- float val = __expf(logits[i] - qk_max);
157
- logits[i] = val;
158
- exp_sum += val;
159
- }
160
- exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
161
-
162
- // Compute softmax.
163
- const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
164
- for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
165
- logits[i] *= inv_sum;
166
- }
167
- __syncthreads();
168
-
169
- // Each thread will fetch 16 bytes from the value cache at a time.
170
- constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
171
- using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
172
- using L_vec = typename FloatVec<V_vec>::Type;
173
-
174
- constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
175
- constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
176
- constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
177
-
178
- float accs[NUM_ROWS_PER_THREAD];
179
-#pragma unroll
180
- for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
181
- accs[i] = 0.f;
182
- }
183
-
184
- for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
185
- const int physical_block_number = block_table[block_idx];
186
- const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
187
- const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
188
- L_vec logits_vec = *reinterpret_cast<L_vec*>(logits + token_idx);
189
-
190
- const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
191
- + head_idx * HEAD_SIZE * BLOCK_SIZE;
192
-#pragma unroll
193
- for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
194
- const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
195
- if (row_idx < HEAD_SIZE) {
196
- const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
197
- V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
198
- accs[i] += dot(logits_vec, cast_to_float(v_vec));
199
- }
200
- }
201
- }
202
-
203
- // Perform reduction within each warp.
204
-#pragma unroll
205
- for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
206
- float acc = accs[i];
207
-#pragma unroll
208
- for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
209
- acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
210
- }
211
- accs[i] = acc;
212
- }
213
-
214
- // NOTE(woosuk): A barrier is required because the shared memory space for logits
215
- // is reused for the output.
216
- __syncthreads();
217
-
218
- // Perform reduction across warps.
219
- float* out_smem = reinterpret_cast<float*>(shared_mem);
220
-#pragma unroll
221
- for (int i = NUM_WARPS; i > 1; i /= 2) {
222
- int mid = i / 2;
223
- // Upper warps write to shared memory.
224
- if (warp_idx >= mid && warp_idx < i) {
225
- float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
226
-#pragma unroll
227
- for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
228
- const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
229
- if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
230
- dst[row_idx] = accs[i];
231
- }
232
- }
233
- }
234
- __syncthreads();
235
-
236
- // Lower warps update the output.
237
- if (warp_idx < mid) {
238
- const float* src = &out_smem[warp_idx * HEAD_SIZE];
239
-#pragma unroll
240
- for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
241
- const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
242
- if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
243
- accs[i] += src[row_idx];
244
- }
245
- }
246
- }
247
- __syncthreads();
248
- }
249
-
250
- // Write the final output.
251
- if (warp_idx == 0) {
252
- scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
253
-#pragma unroll
254
- for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
255
- const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
256
- if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
257
- convert_from_float(*(out_ptr + row_idx), accs[i]);
258
- }
259
- }
260
- }
261
-}
262
-
263
-} // namespace cacheflow
264
-
265
-#define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
266
- cacheflow::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
267
- <<<grid, block, shared_mem_size, stream>>>( \
268
- out_ptr, \
269
- query_ptr, \
270
- key_cache_ptr, \
271
- value_cache_ptr, \
272
- scale, \
273
- block_tables_ptr, \
274
- context_lens_ptr, \
275
- max_num_blocks_per_seq, \
276
- query_stride);
277
-
278
-// TODO(woosuk): Tune NUM_THREADS.
279
-template<
280
- typename T,
281
- int BLOCK_SIZE,
282
- int NUM_THREADS = 128>
283
-void single_query_cached_kv_attention_launcher(
284
- torch::Tensor& out,
285
- torch::Tensor& query,
286
- torch::Tensor& key_cache,
287
- torch::Tensor& value_cache,
288
- float scale,
289
- torch::Tensor& block_tables,
290
- torch::Tensor& context_lens,
291
- int max_context_len) {
292
- int num_seqs = query.size(0);
293
- int num_heads = query.size(1);
294
- int head_size = query.size(2);
295
- int max_num_blocks_per_seq = block_tables.size(1);
296
- int query_stride = query.stride(0);
297
-
298
- int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
299
- assert(head_size % thread_group_size == 0);
300
-
301
- T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
302
- T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
303
- T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
304
- T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
305
- int* block_tables_ptr = block_tables.data_ptr<int>();
306
- int* context_lens_ptr = context_lens.data_ptr<int>();
307
-
308
- constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
309
- int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
310
- int logits_size = padded_max_context_len * sizeof(float);
311
- int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
312
- int shared_mem_size = std::max(logits_size, outputs_size);
313
-
314
- dim3 grid(num_heads, num_seqs);
315
- dim3 block(NUM_THREADS);
316
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
317
- switch (head_size) {
318
- case 32:
319
- LAUNCH_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
320
- break;
321
- case 64:
322
- LAUNCH_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
323
- break;
324
- case 80:
325
- LAUNCH_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
326
- break;
327
- case 96:
328
- LAUNCH_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
329
- break;
330
- case 128:
331
- LAUNCH_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
332
- break;
333
- case 160:
334
- LAUNCH_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
335
- break;
336
- case 192:
337
- LAUNCH_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
338
- break;
339
- case 256:
340
- LAUNCH_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
341
- break;
342
- default:
343
- assert(false);
344
- break;
345
- }
346
-}
347
-
348
-#define CALL_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
349
- single_query_cached_kv_attention_launcher<T, BLOCK_SIZE>( \
350
- out, \
351
- query, \
352
- key_cache, \
353
- value_cache, \
354
- scale, \
355
- block_tables, \
356
- context_lens, \
357
- max_context_len);
358
-
359
-void single_query_cached_kv_attention(
360
- torch::Tensor& out, // [num_seqs, num_heads, head_size]
361
- torch::Tensor& query, // [num_seqs, num_heads, head_size]
362
- torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
363
- torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
364
- float scale,
365
- torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
366
- torch::Tensor& context_lens, // [num_seqs]
367
- int block_size,
368
- int max_context_len) {
369
- // TODO(woosuk): Support BF16.
370
- if (query.element_size() == 2) {
371
- // Half.
372
- if (block_size == 1) {
373
- CALL_KERNEL_LAUNCHER(uint16_t, 1);
374
- } else if (block_size == 2) {
375
- CALL_KERNEL_LAUNCHER(uint16_t, 2);
376
- } else if (block_size == 4) {
377
- CALL_KERNEL_LAUNCHER(uint16_t, 4);
378
- } else if (block_size == 8) {
379
- CALL_KERNEL_LAUNCHER(uint16_t, 8);
380
- } else if (block_size == 16) {
381
- CALL_KERNEL_LAUNCHER(uint16_t, 16);
382
- } else if (block_size == 32) {
383
- CALL_KERNEL_LAUNCHER(uint16_t, 32);
384
- } else if (block_size == 64) {
385
- CALL_KERNEL_LAUNCHER(uint16_t, 64);
386
- } else if (block_size == 128) {
387
- CALL_KERNEL_LAUNCHER(uint16_t, 128);
388
- } else if (block_size == 256) {
389
- CALL_KERNEL_LAUNCHER(uint16_t, 256);
390
- } else {
391
- assert(false);
392
- }
393
- } else {
394
- // Float.
395
- assert(false);
396
- }
397
-}
398
-
399
-// namespace cacheflow {
400
-
401
-// // Grid: (num_heads, num_query_tokens).
402
-// template<
403
-// typename scalar_t,
404
-// int HEAD_SIZE,
405
-// int BLOCK_SIZE,
406
-// int NUM_THREADS>
407
-// __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
408
-// scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
409
-// const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
410
-// const int seq_start_idx,
411
-// const int seq_len,
412
-// const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
413
-// const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
414
-// const float scale,
415
-// const int* __restrict__ block_table, // [num_seqs, max_num_blocks_per_seq]
416
-// const int context_len,
417
-// const int max_num_blocks_per_seq,
418
-// const int q_stride) {
419
-// constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
420
-// constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
421
-// const int thread_idx = threadIdx.x;
422
-// const int warp_idx = thread_idx / WARP_SIZE;
423
-// const int lane = thread_idx % WARP_SIZE;
424
-
425
-// const int head_idx = blockIdx.x;
426
-// const int num_heads = gridDim.x;
427
-// const int seq_idx = blockIdx.y;
428
-
429
-// // A vector type to store a part of a key or a query.
430
-// // The vector size is configured in such a way that the threads in a thread group
431
-// // fetch or comput 16 bytes at a time.
432
-// // For example, if the size of a thread group is 4 and the data type is half,
433
-// // then the vector size is 16 / (4 * sizeof(half)) == 2.
434
-// constexpr int VEC_SIZE = 16 / (THREAD_GROUP_SIZE * sizeof(scalar_t));
435
-// using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
436
-// using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
437
-
438
-// constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
439
-// constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
440
-
441
-// const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
442
-// const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
443
-
444
-// // Load the query to registers.
445
-// // Each thread in a thread group has a different part of the query.
446
-// // For example, if the the thread group size is 4, then the first thread in the group
447
-// // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
448
-// // th vectors of the query, and so on.
449
-// // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
450
-// const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
451
-// Q_vec q_vecs[NUM_VECS_PER_THREAD];
452
-// #pragma unroll
453
-// for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
454
-// const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
455
-// q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
456
-// }
457
-
458
-// // Memory planning.
459
-// extern __shared__ char shared_mem[];
460
-// // NOTE(woosuk): We use FP32 logits and accumulation.
461
-// float *logits = reinterpret_cast<float*>(shared_mem);
462
-// // Workspace for reduction.
463
-// __shared__ float red_smem[2 * NUM_WARPS];
464
-
465
-// // x == THREAD_GROUP_SIZE * VEC_SIZE
466
-// // Each thread group fetches x elements from the key at a time.
467
-// constexpr int x = 16 / sizeof(scalar_t);
468
-// float qk_max = -FLT_MAX;
469
-
470
-// const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
471
-// const int mask_boundary = context_len - seq_len + 1 + (seq_idx - seq_start_idx);
472
-
473
-// // Iterate over the key blocks.
474
-// // Each warp fetches a block of keys for each iteration.
475
-// // Each thread group in a warp fetches a key from the block, and computes
476
-// // dot product with the query.
477
-// for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
478
-// const int physical_block_number = block_table[block_idx];
479
-// const int physical_block_offset = thread_group_idx % BLOCK_SIZE;
480
-// const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
481
-
482
-// // Load a key to registers.
483
-// // Each thread in a thread group has a different part of the key.
484
-// // For example, if the the thread group size is 4, then the first thread in the group
485
-// // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
486
-// // vectors of the key, and so on.
487
-// K_vec k_vecs[NUM_VECS_PER_THREAD];
488
-// #pragma unroll
489
-// for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
490
-// const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
491
-// + head_idx * HEAD_SIZE * BLOCK_SIZE
492
-// + physical_block_offset * x;
493
-// const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
494
-// const int offset1 = (vec_idx * VEC_SIZE) / x;
495
-// const int offset2 = (vec_idx * VEC_SIZE) % x;
496
-// k_vecs[i] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
497
-// }
498
-
499
-// // Compute dot product.
500
-// // This includes a reduction across the threads in the same thread group.
501
-// const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
502
-// const bool mask = token_idx >= mask_boundary;
503
-
504
-// if (thread_group_offset == 0) {
505
-// // Store the partial reductions to shared memory.
506
-// // NOTE(woosuk): It is required to zero out the masked logits.
507
-// logits[token_idx] = mask ? 0.f : qk;
508
-// // Update the max value.
509
-// qk_max = mask ? qk_max : fmaxf(qk_max, qk);
510
-// }
511
-// }
512
-
513
-// // Perform reduction across the threads in the same warp to get the
514
-// // max qk value for each "warp" (not across the thread block yet).
515
-// // The 0-th thread of each thread group already has its max qk value.
516
-// #pragma unroll
517
-// for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
518
-// qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
519
-// }
520
-// if (lane == 0) {
521
-// red_smem[warp_idx] = qk_max;
522
-// }
523
-// __syncthreads();
524
-
525
-// // TODO(woosuk): Refactor this part.
526
-// // Get the max qk value for the sequence.
527
-// qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
528
-// #pragma unroll
529
-// for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
530
-// qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
531
-// }
532
-// // Broadcast the max qk value to all threads.
533
-// qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
534
-
535
-// // Get the sum of the exp values.
536
-// float exp_sum = 0.f;
537
-// for (int i = thread_idx; i < mask_boundary; i += NUM_THREADS) {
538
-// float val = __expf(logits[i] - qk_max);
539
-// logits[i] = val;
540
-// exp_sum += val;
541
-// }
542
-// exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
543
-
544
-// // Compute softmax.
545
-// const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
546
-// for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
547
-// logits[i] *= inv_sum;
548
-// }
549
-// __syncthreads();
550
-
551
-// // Each thread will fetch 16 bytes from the value cache at a time.
552
-// constexpr int V_VEC_SIZE = 16 / sizeof(scalar_t);
553
-// using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
554
-// using L_vec = typename FloatVec<V_vec>::Type;
555
-
556
-// constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
557
-// constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
558
-// constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
559
-
560
-// float accs[NUM_ROWS_PER_THREAD];
561
-// #pragma unroll
562
-// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
563
-// accs[i] = 0.f;
564
-// }
565
-
566
-// for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
567
-// const int physical_block_number = block_table[block_idx];
568
-// const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
569
-// const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
570
-// L_vec logits_vec = *reinterpret_cast<L_vec*>(logits + token_idx);
571
-
572
-// const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
573
-// + head_idx * HEAD_SIZE * BLOCK_SIZE;
574
-// #pragma unroll
575
-// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
576
-// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
577
-// if (row_idx < HEAD_SIZE) {
578
-// const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
579
-// V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
580
-// accs[i] += dot(logits_vec, cast_to_float(v_vec));
581
-// }
582
-// }
583
-// }
584
-
585
-// // Perform reduction within each warp.
586
-// #pragma unroll
587
-// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
588
-// float acc = accs[i];
589
-// #pragma unroll
590
-// for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
591
-// acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
592
-// }
593
-// accs[i] = acc;
594
-// }
595
-
596
-// // NOTE(woosuk): A barrier is required because the shared memory space for logits
597
-// // is reused for the output.
598
-// __syncthreads();
599
-
600
-// // Perform reduction across warps.
601
-// float* out_smem = reinterpret_cast<float*>(shared_mem);
602
-// #pragma unroll
603
-// for (int i = NUM_WARPS; i > 1; i /= 2) {
604
-// int mid = i / 2;
605
-// // Upper warps write to shared memory.
606
-// if (warp_idx >= mid && warp_idx < i) {
607
-// float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
608
-// #pragma unroll
609
-// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
610
-// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
611
-// if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
612
-// dst[row_idx] = accs[i];
613
-// }
614
-// }
615
-// }
616
-// __syncthreads();
617
-
618
-// // Lower warps update the output.
619
-// if (warp_idx < mid) {
620
-// const float* src = &out_smem[warp_idx * HEAD_SIZE];
621
-// #pragma unroll
622
-// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
623
-// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
624
-// if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
625
-// accs[i] += src[row_idx];
626
-// }
627
-// }
628
-// }
629
-// __syncthreads();
630
-// }
631
-
632
-// // Write the final output.
633
-// if (warp_idx == 0) {
634
-// scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
635
-// #pragma unroll
636
-// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
637
-// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
638
-// if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
639
-// convert_from_float(*(out_ptr + row_idx), accs[i]);
640
-// }
641
-// }
642
-// }
643
-// }
644
-
645
-
646
-// // Grid: (num_heads, num_query_tokens).
647
-// template<
648
-// typename scalar_t,
649
-// int HEAD_SIZE,
650
-// int BLOCK_SIZE,
651
-// int NUM_THREADS>
652
-// __global__ void multi_query_cached_kv_attention_kernel(
653
-// const int* cu_query_lens, // [num_prompts+1]
654
-// const int* seq_prompt_mapping, // [num_seqs] mapping from seq_idx to prompt_idx
655
-// scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
656
-// const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
657
-// const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
658
-// const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
659
-// const float scale,
660
-// const int* __restrict__ block_tables, // [num_prompts, max_num_blocks_per_seq]
661
-// const int* __restrict__ context_lens, // [num_prompts]
662
-// const int max_num_blocks_per_seq,
663
-// const int q_stride) {
664
-// const int seq_idx = blockIdx.y;
665
-// const int prompt_idx = seq_prompt_mapping[seq_idx];
666
-// const int seq_start_idx = cu_query_lens[prompt_idx];
667
-// const int seq_len = cu_query_lens[prompt_idx + 1] - seq_start_idx;
668
-// const int* block_table = block_tables + prompt_idx * max_num_blocks_per_seq;
669
-// const int context_len = context_lens[prompt_idx];
670
-// multi_query_cached_kv_attention_kernel_unoptimized_<
671
-// scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
672
-// out,
673
-// q,
674
-// seq_start_idx,
675
-// seq_len,
676
-// k_cache,
677
-// v_cache,
678
-// scale,
679
-// block_table,
680
-// context_len,
681
-// max_num_blocks_per_seq,
682
-// q_stride);
683
-// }
684
-
685
-// } // namespace cacheflow
686
-
687
-// #define LAUNCH_MULTI_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
688
-// cacheflow::multi_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
689
-// <<<grid, block, shared_mem_size, stream>>>( \
690
-// cu_query_lens_ptr, \
691
-// seq_prompt_mapping_ptr, \
692
-// out_ptr, \
693
-// query_ptr, \
694
-// key_cache_ptr, \
695
-// value_cache_ptr, \
696
-// scale, \
697
-// block_tables_ptr, \
698
-// context_lens_ptr, \
699
-// max_num_blocks_per_seq, \
700
-// query_stride);
701
-
702
-
703
-// // TODO(woosuk): Tune NUM_THREADS.
704
-// template<
705
-// typename T,
706
-// int BLOCK_SIZE,
707
-// int NUM_THREADS = 128>
708
-// void multi_query_cached_kv_attention_launcher(
709
-// torch::Tensor& cu_query_lens,
710
-// torch::Tensor& seq_prompt_mapping,
711
-// torch::Tensor& out,
712
-// torch::Tensor& query,
713
-// torch::Tensor& key_cache,
714
-// torch::Tensor& value_cache,
715
-// float scale,
716
-// torch::Tensor& block_tables,
717
-// torch::Tensor& context_lens,
718
-// int max_context_len) {
719
-// int num_seqs = query.size(0);
720
-// int num_heads = query.size(1);
721
-// int head_size = query.size(2);
722
-// int max_num_blocks_per_seq = block_tables.size(1);
723
-// int query_stride = query.stride(0);
724
-
725
-// int* cu_query_lens_ptr = cu_query_lens.data_ptr<int>();
726
-// int* seq_prompt_mapping_ptr = seq_prompt_mapping.data_ptr<int>();
727
-// T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
728
-// T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
729
-// T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
730
-// T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
731
-// int* block_tables_ptr = block_tables.data_ptr<int>();
732
-// int* context_lens_ptr = context_lens.data_ptr<int>();
733
-
734
-// constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
735
-// int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
736
-// int logits_size = padded_max_context_len * sizeof(float);
737
-// int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
738
-// int shared_mem_size = std::max(logits_size, outputs_size);
739
-
740
-// dim3 grid(num_heads, num_seqs);
741
-// dim3 block(NUM_THREADS);
742
-// const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
743
-// switch (head_size) {
744
-// case 32:
745
-// LAUNCH_MULTI_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
746
-// break;
747
-// case 64:
748
-// LAUNCH_MULTI_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
749
-// break;
750
-// case 80:
751
-// LAUNCH_MULTI_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
752
-// break;
753
-// case 96:
754
-// LAUNCH_MULTI_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
755
-// break;
756
-// case 128:
757
-// LAUNCH_MULTI_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
758
-// break;
759
-// case 160:
760
-// LAUNCH_MULTI_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
761
-// break;
762
-// case 192:
763
-// LAUNCH_MULTI_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
764
-// break;
765
-// case 256:
766
-// LAUNCH_MULTI_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
767
-// break;
768
-// default:
769
-// assert(false);
770
-// break;
771
-// }
772
-// }
773
-
774
-// void multi_query_cached_kv_attention(
775
-// torch::Tensor& cu_query_lens,
776
-// torch::Tensor& out,
777
-// torch::Tensor& query,
778
-// torch::Tensor& key_cache,
779
-// torch::Tensor& value_cache,
780
-// float scale,
781
-// torch::Tensor& block_tables,
782
-// torch::Tensor& context_lens,
783
-// int block_size,
784
-// int max_context_len) {
785
-
786
-// torch::Tensor query_lens = cu_query_lens.to(torch::kCPU);
787
-
788
-// int num_queries = query_lens.size(0) - 1;
789
-// const int* query_lens_ptr = query_lens.data_ptr<int>();
790
-// int num_seqs = query.size(0);
791
-
792
-// torch::Tensor cpu_tensor = torch::empty({num_seqs}, torch::dtype(torch::kInt32));
793
-// auto accessor = cpu_tensor.accessor<int32_t, 1>();
794
-// for (int i = 0, query_cursor = 0; i < num_seqs; ++i) {
795
-// if (i >= query_lens_ptr[query_cursor + 1]) {
796
-// ++query_cursor;
797
-// }
798
-// accessor[i] = query_cursor;
799
-// }
800
-
801
-// // TODO(suquark): This can be slow, as it to(torch::kCPU) and to(torch::kCUDA)
802
-// // implicitly synchronizes the CPU and GPU. And we can avoid this issue by giving
803
-// // the mapping as an input parameter. Let's do this optimization in a later PR.
804
-// torch::Tensor seq_prompt_mapping = cpu_tensor.to(torch::kCUDA);
805
-
806
-// // TODO(woosuk): Support BF16.
807
-// if (query.element_size() == 2) {
808
-// // Half.
809
-// if (block_size == 8) {
810
-// multi_query_cached_kv_attention_launcher<uint16_t, 8>(
811
-// cu_query_lens,
812
-// seq_prompt_mapping,
813
-// out,
814
-// query,
815
-// key_cache,
816
-// value_cache,
817
-// scale,
818
-// block_tables,
819
-// context_lens,
820
-// max_context_len);
821
-// } else if (block_size == 16) {
822
-// multi_query_cached_kv_attention_launcher<uint16_t, 16>(
823
-// cu_query_lens,
824
-// seq_prompt_mapping,
825
-// out,
826
-// query,
827
-// key_cache,
828
-// value_cache,
829
-// scale,
830
-// block_tables,
831
-// context_lens,
832
-// max_context_len);
833
-// } else if (block_size == 32) {
834
-// multi_query_cached_kv_attention_launcher<uint16_t, 32>(
835
-// cu_query_lens,
836
-// seq_prompt_mapping,
837
-// out,
838
-// query,
839
-// key_cache,
840
-// value_cache,
841
-// scale,
842
-// block_tables,
843
-// context_lens,
844
-// max_context_len);
845
-// } else {
846
-// assert(false);
847
-// }
848
-// } else if (query.element_size() == 4) {
849
-// // Float.
850
-// if (block_size == 8) {
851
-// multi_query_cached_kv_attention_launcher<float, 8>(
852
-// cu_query_lens,
853
-// seq_prompt_mapping,
854
-// out,
855
-// query,
856
-// key_cache,
857
-// value_cache,
858
-// scale,
859
-// block_tables,
860
-// context_lens,
861
-// max_context_len);
862
-// } else if (block_size == 16) {
863
-// multi_query_cached_kv_attention_launcher<float, 16>(
864
-// cu_query_lens,
865
-// seq_prompt_mapping,
866
-// out,
867
-// query,
868
-// key_cache,
869
-// value_cache,
870
-// scale,
871
-// block_tables,
872
-// context_lens,
873
-// max_context_len);
874
-// } else if (block_size == 32) {
875
-// multi_query_cached_kv_attention_launcher<float, 32>(
876
-// cu_query_lens,
877
-// seq_prompt_mapping,
878
-// out,
879
-// query,
880
-// key_cache,
881
-// value_cache,
882
-// scale,
883
-// block_tables,
884
-// context_lens,
885
-// max_context_len);
886
-// } else {
887
-// assert(false);
888
-// }
889
-// } else {
890
-// assert(false);
891
-// }
892
-// }
893
-
894
-#undef WARP_SIZE
895
-#undef MAX
896
-#undef MIN
csrc/attention_utils.h
deleted
-165
@@ -1,165 +0,0 @@
1
-#pragma once
2
-
3
-#include "cuda_primitives.h"
4
-
5
-#include <float.h>
6
-#include <type_traits>
7
-
8
-#define MMHA_USE_FP32_ACUM_FOR_FMA
9
-#define MMHA_USE_FP32_ACUM_FOR_OUT
10
-
11
-namespace cacheflow {
12
-
13
-// A vector type to store Q, K, V elements.
14
-template<typename T, int VEC_SIZE>
15
-struct Vec {};
16
-template<>
17
-struct Vec<float, 1> {
18
- using Type = float;
19
-};
20
-template<>
21
-struct Vec<float, 2> {
22
- using Type = float2;
23
-};
24
-template<>
25
-struct Vec<float, 4> {
26
- using Type = float4;
27
-};
28
-template<>
29
-struct Vec<uint16_t, 1> {
30
- using Type = uint16_t;
31
-};
32
-template<>
33
-struct Vec<uint16_t, 2> {
34
- using Type = uint32_t;
35
-};
36
-template<>
37
-struct Vec<uint16_t, 4> {
38
- using Type = uint2;
39
-};
40
-template<>
41
-struct Vec<uint16_t, 8> {
42
- using Type = uint4;
43
-};
44
-
45
-template<typename T>
46
-struct FloatVec {};
47
-template<>
48
-struct FloatVec<float> {
49
- using Type = float;
50
-};
51
-template<>
52
-struct FloatVec<float2> {
53
- using Type = float2;
54
-};
55
-template<>
56
-struct FloatVec<float4> {
57
- using Type = float4;
58
-};
59
-template<>
60
-struct FloatVec<uint16_t> {
61
- using Type = float;
62
-};
63
-template<>
64
-struct FloatVec<uint32_t> {
65
- using Type = float2;
66
-};
67
-template<>
68
-struct FloatVec<uint2> {
69
- using Type = Float4_;
70
-};
71
-template<>
72
-struct FloatVec<uint4> {
73
- using Type = Float8_;
74
-};
75
-
76
-template<int THREADS_PER_KEY, typename K_vec, int N>
77
-inline __device__ float qk_dot_(const K_vec (&q)[N], const K_vec (&k)[N])
78
-{
79
- using K_vec_acum = typename FloatVec<K_vec>::Type;
80
- // Compute the parallel products for Q*K^T (treat vector lanes separately).
81
- K_vec_acum qk_vec = mul<K_vec_acum, K_vec, K_vec>(q[0], k[0]);
82
-#pragma unroll
83
- for (int ii = 1; ii < N; ++ii) {
84
- qk_vec = fma(q[ii], k[ii], qk_vec);
85
- }
86
-
87
- // Finalize the reduction across lanes.
88
- float qk = sum(qk_vec);
89
-#pragma unroll
90
- for (int mask = THREADS_PER_KEY / 2; mask >= 1; mask /= 2) {
91
- qk += __shfl_xor_sync(uint32_t(-1), qk, mask);
92
- }
93
- return qk;
94
-}
95
-
96
-////////////////////////////////////////////////////////////////////////////////////////////////////
97
-
98
-template<typename T, int THREADS_PER_KEY>
99
-struct Qk_dot {
100
- template<typename K_vec, int N>
101
- static inline __device__ float dot(const K_vec (&q)[N], const K_vec (&k)[N])
102
- {
103
- return qk_dot_<THREADS_PER_KEY>(q, k);
104
- }
105
-};
106
-
107
-////////////////////////////////////////////////////////////////////////////////////////////////////
108
-
109
-inline __device__ float4 hmma_fp32(const uint2& a, uint32_t b)
110
-{
111
- float4 c;
112
- float zero = 0.f;
113
- asm volatile("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 \n"
114
- " {%0, %1, %2, %3}, \n"
115
- " {%4, %5}, \n"
116
- " {%6}, \n"
117
- " {%7, %7, %7, %7}; \n"
118
-
119
- : "=f"(c.x), "=f"(c.y), "=f"(c.z), "=f"(c.w)
120
- : "r"(a.x) "r"(a.y), "r"(b), "f"(zero));
121
- return c;
122
-}
123
-
124
-////////////////////////////////////////////////////////////////////////////////////////////////////
125
-
126
-template<int N>
127
-inline __device__ float qk_hmma_dot_(const uint32_t (&q)[N], const uint32_t (&k)[N])
128
-{
129
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 750
130
- using K_vec_acum = typename FloatVec<uint32_t>::Type;
131
- K_vec_acum qk_vec = mul<K_vec_acum, uint32_t, uint32_t>(q[0], k[0]);
132
-#pragma unroll
133
- for (int ii = 1; ii < N; ++ii) {
134
- qk_vec = fma(q[ii], k[ii], qk_vec);
135
- }
136
-#ifdef MMHA_USE_FP32_ACUM_FOR_FMA
137
- uint32_t qk_vec_ = float2_to_half2(qk_vec);
138
- return hmma_fp32(make_uint2(qk_vec_, 0u), 0x3c003c00u).x;
139
-#else
140
- return hmma_fp32(make_uint2(qk_vec, 0u), 0x3c003c00u).x;
141
-#endif
142
-#else
143
- return 0.f;
144
-#endif
145
-}
146
-
147
-////////////////////////////////////////////////////////////////////////////////////////////////////
148
-
149
-template<>
150
-struct Qk_dot<uint16_t, 4> {
151
- template<int N>
152
- static inline __device__ float dot(const uint32_t (&q)[N], const uint32_t (&k)[N])
153
- {
154
-#if __CUDA_ARCH__ >= 750 && defined(MMHA_USE_HMMA_FOR_REDUCTION)
155
- return qk_hmma_dot_(q, k);
156
-#else
157
- return qk_dot_<4>(q, k);
158
-#endif // defined MMHA_USE_HMMA_FOR_REDUCTION
159
- }
160
-};
161
-
162
-} // namespace cacheflow
163
-
164
-#undef MMHA_USE_FP32_ACUM_FOR_FMA
165
-#undef MMHA_USE_FP32_ACUM_FOR_OUT
csrc/cuda_primitives.h
deleted
-1340
@@ -1,1340 +0,0 @@
1
-#pragma once
2
-
3
-#include <stdint.h>
4
-
5
-namespace cacheflow {
6
-////////////////////////////////////////////////////////////////////////////////////////////////////
7
-
8
-struct Float8_ {
9
- float2 x;
10
- float2 y;
11
- float2 z;
12
- float2 w;
13
-};
14
-
15
-////////////////////////////////////////////////////////////////////////////////////////////////////
16
-
17
-struct Float4_ {
18
- float2 x;
19
- float2 y;
20
-};
21
-
22
-////////////////////////////////////////////////////////////////////////////////////////////////////
23
-
24
-#ifdef ENABLE_BF16
25
-struct bf16_4_t {
26
- __nv_bfloat162 x;
27
- __nv_bfloat162 y;
28
-};
29
-
30
-////////////////////////////////////////////////////////////////////////////////////////////////////
31
-
32
-struct bf16_8_t {
33
- __nv_bfloat162 x;
34
- __nv_bfloat162 y;
35
- __nv_bfloat162 z;
36
- __nv_bfloat162 w;
37
-};
38
-#endif
39
-
40
-////////////////////////////////////////////////////////////////////////////////////////////////////
41
-
42
-inline __device__ float add(float a, float b)
43
-{
44
- return a + b;
45
-}
46
-
47
-////////////////////////////////////////////////////////////////////////////////////////////////////
48
-
49
-inline __device__ float2 add(float2 a, float2 b)
50
-{
51
- float2 c;
52
- c.x = add(a.x, b.x);
53
- c.y = add(a.y, b.y);
54
- return c;
55
-}
56
-
57
-////////////////////////////////////////////////////////////////////////////////////////////////////
58
-
59
-inline __device__ float4 add(float4 a, float4 b)
60
-{
61
- float4 c;
62
- c.x = add(a.x, b.x);
63
- c.y = add(a.y, b.y);
64
- c.z = add(a.z, b.z);
65
- c.w = add(a.w, b.w);
66
- return c;
67
-}
68
-
69
-////////////////////////////////////////////////////////////////////////////////////////////////////
70
-
71
-#ifdef ENABLE_BF16
72
-inline __device__ __nv_bfloat16 add(__nv_bfloat16 a, __nv_bfloat16 b)
73
-{
74
- return a + b;
75
-}
76
-
77
-////////////////////////////////////////////////////////////////////////////////////////////////////
78
-
79
-inline __device__ __nv_bfloat162 add(__nv_bfloat162 a, __nv_bfloat162 b)
80
-{
81
- return bf16hadd2(a, b);
82
-}
83
-
84
-////////////////////////////////////////////////////////////////////////////////////////////////////
85
-
86
-inline __device__ bf16_4_t add(bf16_4_t a, bf16_4_t b)
87
-{
88
- bf16_4_t c;
89
- c.x = add(a.x, b.x);
90
- c.y = add(a.y, b.y);
91
- return c;
92
-}
93
-
94
-////////////////////////////////////////////////////////////////////////////////////////////////////
95
-
96
-inline __device__ bf16_8_t add(bf16_8_t a, bf16_8_t b)
97
-{
98
- bf16_8_t c;
99
- c.x = add(a.x, b.x);
100
- c.y = add(a.y, b.y);
101
- c.z = add(a.z, b.z);
102
- c.w = add(a.w, b.w);
103
- return c;
104
-}
105
-#endif // ENABLE_BF16
106
-
107
-////////////////////////////////////////////////////////////////////////////////////////////////////
108
-
109
-inline __device__ uint16_t add(uint16_t a, uint16_t b)
110
-{
111
- uint16_t c;
112
- asm volatile("add.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
113
- return c;
114
-}
115
-
116
-////////////////////////////////////////////////////////////////////////////////////////////////////
117
-
118
-inline __device__ uint32_t add(uint32_t a, uint32_t b)
119
-{
120
- uint32_t c;
121
- asm volatile("add.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
122
- return c;
123
-}
124
-
125
-////////////////////////////////////////////////////////////////////////////////////////////////////
126
-
127
-inline __device__ uint2 add(uint2 a, uint2 b)
128
-{
129
- uint2 c;
130
- c.x = add(a.x, b.x);
131
- c.y = add(a.y, b.y);
132
- return c;
133
-}
134
-
135
-////////////////////////////////////////////////////////////////////////////////////////////////////
136
-
137
-inline __device__ uint4 add(uint4 a, uint4 b)
138
-{
139
- uint4 c;
140
- c.x = add(a.x, b.x);
141
- c.y = add(a.y, b.y);
142
- c.z = add(a.z, b.z);
143
- c.w = add(a.w, b.w);
144
- return c;
145
-}
146
-
147
-////////////////////////////////////////////////////////////////////////////////////////////////////
148
-
149
-inline __device__ uint16_t float_to_half(float f)
150
-{
151
- union {
152
- uint32_t u32;
153
- uint16_t u16[2];
154
- } tmp;
155
-#if 0 && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 // Is it better?
156
- float zero = 0.f;
157
- asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(zero), "f"(f));
158
-#else
159
- asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f));
160
-#endif
161
- return tmp.u16[0];
162
-}
163
-
164
-////////////////////////////////////////////////////////////////////////////////////////////////////
165
-
166
-inline __device__ uint32_t float2_to_half2(float2 f)
167
-{
168
- union {
169
- uint32_t u32;
170
- uint16_t u16[2];
171
- } tmp;
172
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
173
- asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(f.y), "f"(f.x));
174
-#else
175
- asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
176
- asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
177
-#endif
178
- return tmp.u32;
179
-}
180
-
181
-////////////////////////////////////////////////////////////////////////////////////////////////////
182
-
183
-inline __device__ float half_to_float(uint16_t h)
184
-{
185
- float f;
186
- asm volatile("cvt.f32.f16 %0, %1;\n" : "=f"(f) : "h"(h));
187
- return f;
188
-}
189
-
190
-////////////////////////////////////////////////////////////////////////////////////////////////////
191
-
192
-inline __device__ float2 half2_to_float2(uint32_t v)
193
-{
194
- uint16_t lo, hi;
195
- asm volatile("mov.b32 {%0, %1}, %2;\n" : "=h"(lo), "=h"(hi) : "r"(v));
196
- return make_float2(half_to_float(lo), half_to_float(hi));
197
-}
198
-
199
-////////////////////////////////////////////////////////////////////////////////////////////////////
200
-
201
-inline __device__ float2 add(uint32_t a, float2 fb)
202
-{
203
- float2 fa = half2_to_float2(a);
204
- return add(fa, fb);
205
-}
206
-
207
-////////////////////////////////////////////////////////////////////////////////////////////////////
208
-
209
-inline __device__ Float4_ add(uint2 a, Float4_ fb)
210
-{
211
- Float4_ fc;
212
- fc.x = add(a.x, fb.x);
213
- fc.y = add(a.y, fb.y);
214
- return fc;
215
-}
216
-
217
-////////////////////////////////////////////////////////////////////////////////////////////////////
218
-
219
-inline __device__ Float8_ add(uint4 a, Float8_ fb)
220
-{
221
- Float8_ fc;
222
- fc.x = add(a.x, fb.x);
223
- fc.y = add(a.y, fb.y);
224
- fc.z = add(a.z, fb.z);
225
- fc.w = add(a.w, fb.w);
226
- return fc;
227
-}
228
-
229
-////////////////////////////////////////////////////////////////////////////////////////////////////
230
-
231
-inline __device__ uint32_t h0_h0(uint16_t a)
232
-{
233
- uint32_t b;
234
- asm volatile("mov.b32 %0, {%1, %1};" : "=r"(b) : "h"(a));
235
- return b;
236
-}
237
-
238
-////////////////////////////////////////////////////////////////////////////////////////////////////
239
-
240
-inline __device__ float fma(float a, float b, float c)
241
-{
242
- return a * b + c;
243
-}
244
-
245
-////////////////////////////////////////////////////////////////////////////////////////////////////
246
-
247
-inline __device__ float2 fma(float2 a, float2 b, float2 c)
248
-{
249
- float2 d;
250
- d.x = fma(a.x, b.x, c.x);
251
- d.y = fma(a.y, b.y, c.y);
252
- return d;
253
-}
254
-
255
-////////////////////////////////////////////////////////////////////////////////////////////////////
256
-
257
-inline __device__ float2 fma(float a, float2 b, float2 c)
258
-{
259
- float2 d;
260
- d.x = fma(a, b.x, c.x);
261
- d.y = fma(a, b.y, c.y);
262
- return d;
263
-}
264
-
265
-////////////////////////////////////////////////////////////////////////////////////////////////////
266
-
267
-inline __device__ float4 fma(float4 a, float4 b, float4 c)
268
-{
269
- float4 d;
270
- d.x = fma(a.x, b.x, c.x);
271
- d.y = fma(a.y, b.y, c.y);
272
- d.z = fma(a.z, b.z, c.z);
273
- d.w = fma(a.w, b.w, c.w);
274
- return d;
275
-}
276
-
277
-////////////////////////////////////////////////////////////////////////////////////////////////////
278
-
279
-inline __device__ float4 fma(float a, float4 b, float4 c)
280
-{
281
- float4 d;
282
- d.x = fma(a, b.x, c.x);
283
- d.y = fma(a, b.y, c.y);
284
- d.z = fma(a, b.z, c.z);
285
- d.w = fma(a, b.w, c.w);
286
- return d;
287
-}
288
-
289
-////////////////////////////////////////////////////////////////////////////////////////////////////
290
-
291
-inline __device__ Float4_ fma(float a, Float4_ b, Float4_ c)
292
-{
293
- Float4_ d;
294
- d.x = fma(a, b.x, c.x);
295
- d.y = fma(a, b.y, c.y);
296
- return d;
297
-}
298
-
299
-////////////////////////////////////////////////////////////////////////////////////////////////////
300
-
301
-inline __device__ Float8_ fma(float a, Float8_ b, Float8_ c)
302
-{
303
- Float8_ d;
304
- d.x = fma(a, b.x, c.x);
305
- d.y = fma(a, b.y, c.y);
306
- d.z = fma(a, b.z, c.z);
307
- d.w = fma(a, b.w, c.w);
308
- return d;
309
-}
310
-
311
-////////////////////////////////////////////////////////////////////////////////////////////////////
312
-
313
-#ifdef ENABLE_BF16
314
-inline __device__ float2 add(__nv_bfloat162 a, float2 fb)
315
-{
316
- float2 fa = bf1622float2(a);
317
- return add(fa, fb);
318
-}
319
-
320
-////////////////////////////////////////////////////////////////////////////////////////////////////
321
-
322
-inline __device__ Float4_ add(bf16_4_t a, Float4_ fb)
323
-{
324
- Float4_ fc;
325
- fc.x = add(a.x, fb.x);
326
- fc.y = add(a.y, fb.y);
327
- return fc;
328
-}
329
-
330
-////////////////////////////////////////////////////////////////////////////////////////////////////
331
-
332
-inline __device__ Float8_ add(bf16_8_t a, Float8_ fb)
333
-{
334
- Float8_ fc;
335
- fc.x = add(a.x, fb.x);
336
- fc.y = add(a.y, fb.y);
337
- fc.z = add(a.z, fb.z);
338
- fc.w = add(a.w, fb.w);
339
- return fc;
340
-}
341
-#endif // ENABLE_BF16
342
-
343
-////////////////////////////////////////////////////////////////////////////////////////////////////
344
-
345
-inline __device__ uint32_t fma(uint32_t a, uint32_t b, uint32_t c)
346
-{
347
- uint32_t d;
348
- asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(d) : "r"(a), "r"(b), "r"(c));
349
- return d;
350
-}
351
-
352
-////////////////////////////////////////////////////////////////////////////////////////////////////
353
-
354
-inline __device__ uint32_t fma(uint16_t a, uint32_t b, uint32_t c)
355
-{
356
- return fma(h0_h0(a), b, c);
357
-}
358
-
359
-////////////////////////////////////////////////////////////////////////////////////////////////////
360
-
361
-inline __device__ uint2 fma(uint2 a, uint2 b, uint2 c)
362
-{
363
- uint2 d;
364
- d.x = fma(a.x, b.x, c.x);
365
- d.y = fma(a.y, b.y, c.y);
366
- return d;
367
-}
368
-
369
-////////////////////////////////////////////////////////////////////////////////////////////////////
370
-
371
-inline __device__ uint2 fma(uint16_t a, uint2 b, uint2 c)
372
-{
373
- uint32_t s = h0_h0(a);
374
- uint2 d;
375
- d.x = fma(s, b.x, c.x);
376
- d.y = fma(s, b.y, c.y);
377
- return d;
378
-}
379
-
380
-////////////////////////////////////////////////////////////////////////////////////////////////////
381
-
382
-inline __device__ uint4 fma(uint4 a, uint4 b, uint4 c)
383
-{
384
- uint4 d;
385
- d.x = fma(a.x, b.x, c.x);
386
- d.y = fma(a.y, b.y, c.y);
387
- d.z = fma(a.z, b.z, c.z);
388
- d.w = fma(a.w, b.w, c.w);
389
- return d;
390
-}
391
-
392
-////////////////////////////////////////////////////////////////////////////////////////////////////
393
-
394
-inline __device__ uint4 fma(uint16_t a, uint4 b, uint4 c)
395
-{
396
- uint32_t s = h0_h0(a);
397
- uint4 d;
398
- d.x = fma(s, b.x, c.x);
399
- d.y = fma(s, b.y, c.y);
400
- d.z = fma(s, b.z, c.z);
401
- d.w = fma(s, b.w, c.w);
402
- return d;
403
-}
404
-
405
-////////////////////////////////////////////////////////////////////////////////////////////////////
406
-
407
-inline __device__ float fma(uint16_t a, uint16_t b, float fc)
408
-{
409
- float fa = half_to_float(a);
410
- float fb = half_to_float(b);
411
- return fa * fb + fc;
412
-}
413
-
414
-////////////////////////////////////////////////////////////////////////////////////////////////////
415
-
416
-inline __device__ float2 fma(uint32_t a, uint32_t b, float2 fc)
417
-{
418
- float2 fa = half2_to_float2(a);
419
- float2 fb = half2_to_float2(b);
420
- return fma(fa, fb, fc);
421
-}
422
-
423
-////////////////////////////////////////////////////////////////////////////////////////////////////
424
-
425
-inline __device__ float2 fma(uint16_t a, uint32_t b, float2 fc)
426
-{
427
- return fma(h0_h0(a), b, fc);
428
-}
429
-
430
-////////////////////////////////////////////////////////////////////////////////////////////////////
431
-
432
-inline __device__ Float4_ fma(uint2 a, uint2 b, Float4_ fc)
433
-{
434
- Float4_ fd;
435
- fd.x = fma(a.x, b.x, fc.x);
436
- fd.y = fma(a.y, b.y, fc.y);
437
- return fd;
438
-}
439
-
440
-////////////////////////////////////////////////////////////////////////////////////////////////////
441
-
442
-inline __device__ Float4_ fma(uint16_t a, uint2 b, Float4_ fc)
443
-{
444
- uint32_t s = h0_h0(a);
445
- Float4_ fd;
446
- fd.x = fma(s, b.x, fc.x);
447
- fd.y = fma(s, b.y, fc.y);
448
- return fd;
449
-}
450
-
451
-////////////////////////////////////////////////////////////////////////////////////////////////////
452
-
453
-inline __device__ Float8_ fma(uint4 a, uint4 b, Float8_ fc)
454
-{
455
- Float8_ fd;
456
- fd.x = fma(a.x, b.x, fc.x);
457
- fd.y = fma(a.y, b.y, fc.y);
458
- fd.z = fma(a.z, b.z, fc.z);
459
- fd.w = fma(a.w, b.w, fc.w);
460
- return fd;
461
-}
462
-
463
-////////////////////////////////////////////////////////////////////////////////////////////////////
464
-
465
-inline __device__ Float8_ fma(uint16_t a, uint4 b, Float8_ fc)
466
-{
467
- uint32_t s = h0_h0(a);
468
- Float8_ fd;
469
- fd.x = fma(s, b.x, fc.x);
470
- fd.y = fma(s, b.y, fc.y);
471
- fd.z = fma(s, b.z, fc.z);
472
- fd.w = fma(s, b.w, fc.w);
473
- return fd;
474
-}
475
-
476
-////////////////////////////////////////////////////////////////////////////////////////////////////
477
-#ifdef ENABLE_BF16
478
-inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b, __nv_bfloat162 c)
479
-{
480
- return bf16hfma2(a, b, c);
481
-}
482
-
483
-////////////////////////////////////////////////////////////////////////////////////////////////////
484
-
485
-inline __device__ __nv_bfloat162 fma(__nv_bfloat16 a, __nv_bfloat162 b, __nv_bfloat162 c)
486
-{
487
- return bf16hfma2(bf162bf162(a), b, c);
488
-}
489
-////////////////////////////////////////////////////////////////////////////////////////////////////
490
-
491
-inline __device__ bf16_4_t fma(bf16_4_t a, bf16_4_t b, bf16_4_t c)
492
-{
493
- bf16_4_t d;
494
- d.x = fma(a.x, b.x, c.x);
495
- d.y = fma(a.y, b.y, c.y);
496
- return d;
497
-}
498
-
499
-////////////////////////////////////////////////////////////////////////////////////////////////////
500
-
501
-inline __device__ bf16_4_t fma(__nv_bfloat16 a, bf16_4_t b, bf16_4_t c)
502
-{
503
- __nv_bfloat162 s = bf162bf162(a);
504
- bf16_4_t d;
505
- d.x = fma(s, b.x, c.x);
506
- d.y = fma(s, b.y, c.y);
507
- return d;
508
-}
509
-
510
-////////////////////////////////////////////////////////////////////////////////////////////////////
511
-
512
-inline __device__ bf16_8_t fma(bf16_8_t a, bf16_8_t b, bf16_8_t c)
513
-{
514
- bf16_8_t d;
515
- d.x = fma(a.x, b.x, c.x);
516
- d.y = fma(a.y, b.y, c.y);
517
- d.z = fma(a.z, b.z, c.z);
518
- d.w = fma(a.w, b.w, c.w);
519
- return d;
520
-}
521
-
522
-////////////////////////////////////////////////////////////////////////////////////////////////////
523
-
524
-inline __device__ bf16_8_t fma(__nv_bfloat16 a, bf16_8_t b, bf16_8_t c)
525
-{
526
- __nv_bfloat162 s = bf162bf162(a);
527
- bf16_8_t d;
528
- d.x = fma(s, b.x, c.x);
529
- d.y = fma(s, b.y, c.y);
530
- d.z = fma(s, b.z, c.z);
531
- d.w = fma(s, b.w, c.w);
532
- return d;
533
-}
534
-
535
-////////////////////////////////////////////////////////////////////////////////////////////////////
536
-
537
-inline __device__ float fma(__nv_bfloat16 a, __nv_bfloat16 b, float fc)
538
-{
539
- return __bfloat162float(a) * __bfloat162float(b) + fc;
540
-}
541
-
542
-////////////////////////////////////////////////////////////////////////////////////////////////////
543
-
544
-inline __device__ float2 fma(__nv_bfloat162 a, __nv_bfloat162 b, float2 fc)
545
-{
546
- float2 fa = bf1622float2(a);
547
- float2 fb = bf1622float2(b);
548
- return fma(fa, fb, fc);
549
-}
550
-
551
-////////////////////////////////////////////////////////////////////////////////////////////////////
552
-
553
-inline __device__ float2 fma(__nv_bfloat16 a, __nv_bfloat162 b, float2 fc)
554
-{
555
- return fma(bf162bf162(a), b, fc);
556
-}
557
-
558
-////////////////////////////////////////////////////////////////////////////////////////////////////
559
-
560
-inline __device__ Float4_ fma(bf16_4_t a, bf16_4_t b, Float4_ fc)
561
-{
562
- Float4_ fd;
563
- fd.x = fma(a.x, b.x, fc.x);
564
- fd.y = fma(a.y, b.y, fc.y);
565
- return fd;
566
-}
567
-
568
-////////////////////////////////////////////////////////////////////////////////////////////////////
569
-
570
-inline __device__ Float4_ fma(__nv_bfloat16 a, bf16_4_t b, Float4_ fc)
571
-{
572
- __nv_bfloat162 s = bf162bf162(a);
573
- Float4_ fd;
574
- fd.x = fma(s, b.x, fc.x);
575
- fd.y = fma(s, b.y, fc.y);
576
- return fd;
577
-}
578
-
579
-////////////////////////////////////////////////////////////////////////////////////////////////////
580
-
581
-inline __device__ Float8_ fma(bf16_8_t a, bf16_8_t b, Float8_ fc)
582
-{
583
- Float8_ fd;
584
- fd.x = fma(a.x, b.x, fc.x);
585
- fd.y = fma(a.y, b.y, fc.y);
586
- fd.z = fma(a.z, b.z, fc.z);
587
- fd.w = fma(a.w, b.w, fc.w);
588
- return fd;
589
-}
590
-
591
-////////////////////////////////////////////////////////////////////////////////////////////////////
592
-
593
-inline __device__ Float8_ fma(__nv_bfloat16 a, bf16_8_t b, Float8_ fc)
594
-{
595
- __nv_bfloat162 s = bf162bf162(a);
596
- Float8_ fd;
597
- fd.x = fma(s, b.x, fc.x);
598
- fd.y = fma(s, b.y, fc.y);
599
- fd.z = fma(s, b.z, fc.z);
600
- fd.w = fma(s, b.w, fc.w);
601
- return fd;
602
-}
603
-#endif // ENABLE_BF16
604
-////////////////////////////////////////////////////////////////////////////////////////////////////
605
-
606
-template<typename Acc, typename A, typename B>
607
-inline __device__ Acc mul(A a, B b);
608
-
609
-////////////////////////////////////////////////////////////////////////////////////////////////////
610
-
611
-template<>
612
-inline __device__ float mul<float, float>(float a, float b)
613
-{
614
- return a * b;
615
-}
616
-
617
-////////////////////////////////////////////////////////////////////////////////////////////////////
618
-
619
-template<>
620
-inline __device__ float2 mul(float2 a, float2 b)
621
-{
622
- float2 c;
623
- c.x = a.x * b.x;
624
- c.y = a.y * b.y;
625
- return c;
626
-}
627
-
628
-////////////////////////////////////////////////////////////////////////////////////////////////////
629
-
630
-template<>
631
-inline __device__ float2 mul(float a, float2 b)
632
-{
633
- float2 c;
634
- c.x = a * b.x;
635
- c.y = a * b.y;
636
- return c;
637
-}
638
-
639
-////////////////////////////////////////////////////////////////////////////////////////////////////
640
-
641
-template<>
642
-inline __device__ float4 mul(float4 a, float4 b)
643
-{
644
- float4 c;
645
- c.x = a.x * b.x;
646
- c.y = a.y * b.y;
647
- c.z = a.z * b.z;
648
- c.w = a.w * b.w;
649
- return c;
650
-}
651
-
652
-////////////////////////////////////////////////////////////////////////////////////////////////////
653
-
654
-template<>
655
-inline __device__ float4 mul(float a, float4 b)
656
-{
657
- float4 c;
658
- c.x = a * b.x;
659
- c.y = a * b.y;
660
- c.z = a * b.z;
661
- c.w = a * b.w;
662
- return c;
663
-}
664
-
665
-////////////////////////////////////////////////////////////////////////////////////////////////////
666
-
667
-template<>
668
-inline __device__ uint16_t mul(uint16_t a, uint16_t b)
669
-{
670
- uint16_t c;
671
- asm volatile("mul.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
672
- return c;
673
-}
674
-
675
-////////////////////////////////////////////////////////////////////////////////////////////////////
676
-
677
-template<>
678
-inline __device__ uint32_t mul(uint32_t a, uint32_t b)
679
-{
680
- uint32_t c;
681
- asm volatile("mul.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
682
- return c;
683
-}
684
-
685
-////////////////////////////////////////////////////////////////////////////////////////////////////
686
-
687
-template<>
688
-inline __device__ uint32_t mul(uint16_t a, uint32_t b)
689
-{
690
- return mul<uint32_t, uint32_t, uint32_t>(h0_h0(a), b);
691
-}
692
-
693
-////////////////////////////////////////////////////////////////////////////////////////////////////
694
-
695
-template<>
696
-inline __device__ uint2 mul(uint2 a, uint2 b)
697
-{
698
- uint2 c;
699
- c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
700
- c.y = mul<uint32_t, uint32_t, uint32_t>(a.y, b.y);
701
- return c;
702
-}
703
-
704
-////////////////////////////////////////////////////////////////////////////////////////////////////
705
-
706
-template<>
707
-inline __device__ uint2 mul(uint16_t a, uint2 b)
708
-{
709
- uint32_t s = h0_h0(a);
710
- uint2 c;
711
- c.x = mul<uint32_t, uint32_t, uint32_t>(s, b.x);
712
- c.y = mul<uint32_t, uint32_t, uint32_t>(s, b.y);
713
- return c;
714
-}
715
-
716
-////////////////////////////////////////////////////////////////////////////////////////////////////
717
-
718
-template<>
719
-inline __device__ uint4 mul(uint4 a, uint4 b)
720
-{
721
- uint4 c;
722
- c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
723
- c.y = mul<uint32_t, uint32_t, uint32_t>(a.y, b.y);
724
- c.z = mul<uint32_t, uint32_t, uint32_t>(a.z, b.z);
725
- c.w = mul<uint32_t, uint32_t, uint32_t>(a.w, b.w);
726
- return c;
727
-}
728
-
729
-////////////////////////////////////////////////////////////////////////////////////////////////////
730
-
731
-template<>
732
-inline __device__ uint4 mul(uint16_t a, uint4 b)
733
-{
734
- uint32_t s = h0_h0(a);
735
- uint4 c;
736
- c.x = mul<uint32_t, uint32_t, uint32_t>(s, b.x);
737
- c.y = mul<uint32_t, uint32_t, uint32_t>(s, b.y);
738
- c.z = mul<uint32_t, uint32_t, uint32_t>(s, b.z);
739
- c.w = mul<uint32_t, uint32_t, uint32_t>(s, b.w);
740
- return c;
741
-}
742
-
743
-////////////////////////////////////////////////////////////////////////////////////////////////////
744
-
745
-template<>
746
-inline __device__ float mul(uint16_t a, uint16_t b)
747
-{
748
- float fa = half_to_float(a);
749
- float fb = half_to_float(b);
750
- return fa * fb;
751
-}
752
-
753
-////////////////////////////////////////////////////////////////////////////////////////////////////
754
-
755
-template<>
756
-inline __device__ float2 mul(uint32_t a, uint32_t b)
757
-{
758
- float2 fa = half2_to_float2(a);
759
- float2 fb = half2_to_float2(b);
760
- return mul<float2, float2, float2>(fa, fb);
761
-}
762
-
763
-////////////////////////////////////////////////////////////////////////////////////////////////////
764
-
765
-template<>
766
-inline __device__ float2 mul(uint16_t a, uint32_t b)
767
-{
768
- return mul<float2, uint32_t, uint32_t>(h0_h0(a), b);
769
-}
770
-
771
-////////////////////////////////////////////////////////////////////////////////////////////////////
772
-
773
-template<>
774
-inline __device__ Float4_ mul(uint2 a, uint2 b)
775
-{
776
- Float4_ fc;
777
- fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
778
- fc.y = mul<float2, uint32_t, uint32_t>(a.y, b.y);
779
- return fc;
780
-}
781
-
782
-////////////////////////////////////////////////////////////////////////////////////////////////////
783
-
784
-template<>
785
-inline __device__ Float4_ mul(uint16_t a, uint2 b)
786
-{
787
- uint32_t s = h0_h0(a);
788
- Float4_ fc;
789
- fc.x = mul<float2, uint32_t, uint32_t>(s, b.x);
790
- fc.y = mul<float2, uint32_t, uint32_t>(s, b.y);
791
- return fc;
792
-}
793
-
794
-////////////////////////////////////////////////////////////////////////////////////////////////////
795
-
796
-template<>
797
-inline __device__ Float8_ mul(uint4 a, uint4 b)
798
-{
799
- Float8_ fc;
800
- fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
801
- fc.y = mul<float2, uint32_t, uint32_t>(a.y, b.y);
802
- fc.z = mul<float2, uint32_t, uint32_t>(a.z, b.z);
803
- fc.w = mul<float2, uint32_t, uint32_t>(a.w, b.w);
804
- return fc;
805
-}
806
-
807
-////////////////////////////////////////////////////////////////////////////////////////////////////
808
-
809
-template<>
810
-inline __device__ Float8_ mul(uint16_t a, uint4 b)
811
-{
812
- uint32_t s = h0_h0(a);
813
- Float8_ fc;
814
- fc.x = mul<float2, uint32_t, uint32_t>(s, b.x);
815
- fc.y = mul<float2, uint32_t, uint32_t>(s, b.y);
816
- fc.z = mul<float2, uint32_t, uint32_t>(s, b.z);
817
- fc.w = mul<float2, uint32_t, uint32_t>(s, b.w);
818
- return fc;
819
-}
820
-
821
-////////////////////////////////////////////////////////////////////////////////////////////////////
822
-
823
-#ifdef ENABLE_BF16
824
-template<>
825
-inline __device__ __nv_bfloat16 mul(__nv_bfloat16 a, __nv_bfloat16 b)
826
-{
827
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
828
- return __hmul(a, b);
829
-#else
830
- return bf16hmul(a, b);
831
-#endif
832
-}
833
-
834
-////////////////////////////////////////////////////////////////////////////////////////////////////
835
-
836
-template<>
837
-inline __device__ __nv_bfloat162 mul(__nv_bfloat162 a, __nv_bfloat162 b)
838
-{
839
- return bf16hmul2(a, b);
840
-}
841
-
842
-////////////////////////////////////////////////////////////////////////////////////////////////////
843
-
844
-template<>
845
-inline __device__ __nv_bfloat162 mul(__nv_bfloat16 a, __nv_bfloat162 b)
846
-{
847
- return mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
848
-}
849
-
850
-////////////////////////////////////////////////////////////////////////////////////////////////////
851
-
852
-template<>
853
-inline __device__ bf16_4_t mul(bf16_4_t a, bf16_4_t b)
854
-{
855
- bf16_4_t c;
856
- c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
857
- c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
858
- return c;
859
-}
860
-
861
-////////////////////////////////////////////////////////////////////////////////////////////////////
862
-
863
-template<>
864
-inline __device__ bf16_4_t mul(__nv_bfloat16 a, bf16_4_t b)
865
-{
866
- __nv_bfloat162 s = bf162bf162(a);
867
- bf16_4_t c;
868
- c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.x);
869
- c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.y);
870
- return c;
871
-}
872
-
873
-////////////////////////////////////////////////////////////////////////////////////////////////////
874
-
875
-template<>
876
-inline __device__ bf16_8_t mul(bf16_8_t a, bf16_8_t b)
877
-{
878
- bf16_8_t c;
879
- c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
880
- c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
881
- c.z = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.z, b.z);
882
- c.w = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.w, b.w);
883
- return c;
884
-}
885
-
886
-////////////////////////////////////////////////////////////////////////////////////////////////////
887
-
888
-template<>
889
-inline __device__ bf16_8_t mul(__nv_bfloat16 a, bf16_8_t b)
890
-{
891
- __nv_bfloat162 s = bf162bf162(a);
892
- bf16_8_t c;
893
- c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.x);
894
- c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.y);
895
- c.z = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.z);
896
- c.w = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.w);
897
- return c;
898
-}
899
-
900
-////////////////////////////////////////////////////////////////////////////////////////////////////
901
-
902
-template<>
903
-inline __device__ float mul(__nv_bfloat16 a, __nv_bfloat16 b)
904
-{
905
- float fa = (float)a;
906
- float fb = (float)b;
907
- return fa * fb;
908
-}
909
-
910
-////////////////////////////////////////////////////////////////////////////////////////////////////
911
-
912
-template<>
913
-inline __device__ float2 mul(__nv_bfloat162 a, __nv_bfloat162 b)
914
-{
915
- float2 fa = bf1622float2(a);
916
- float2 fb = bf1622float2(b);
917
- return mul<float2, float2, float2>(fa, fb);
918
-}
919
-
920
-////////////////////////////////////////////////////////////////////////////////////////////////////
921
-
922
-template<>
923
-inline __device__ float2 mul(__nv_bfloat16 a, __nv_bfloat162 b)
924
-{
925
- return mul<float2, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
926
-}
927
-
928
-////////////////////////////////////////////////////////////////////////////////////////////////////
929
-
930
-template<>
931
-inline __device__ Float4_ mul(bf16_4_t a, bf16_4_t b)
932
-{
933
- Float4_ fc;
934
- fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
935
- fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
936
- return fc;
937
-}
938
-
939
-////////////////////////////////////////////////////////////////////////////////////////////////////
940
-
941
-template<>
942
-inline __device__ Float4_ mul(__nv_bfloat16 a, bf16_4_t b)
943
-{
944
- __nv_bfloat162 s = bf162bf162(a);
945
- Float4_ fc;
946
- fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.x);
947
- fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.y);
948
- return fc;
949
-}
950
-
951
-////////////////////////////////////////////////////////////////////////////////////////////////////
952
-
953
-template<>
954
-inline __device__ Float8_ mul(bf16_8_t a, bf16_8_t b)
955
-{
956
- Float8_ fc;
957
- fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
958
- fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
959
- fc.z = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.z, b.z);
960
- fc.w = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.w, b.w);
961
- return fc;
962
-}
963
-
964
-////////////////////////////////////////////////////////////////////////////////////////////////////
965
-
966
-template<>
967
-inline __device__ Float8_ mul(__nv_bfloat16 a, bf16_8_t b)
968
-{
969
- __nv_bfloat162 s = bf162bf162(a);
970
- Float8_ fc;
971
- fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.x);
972
- fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.y);
973
- fc.z = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.z);
974
- fc.w = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.w);
975
- return fc;
976
-}
977
-#endif // ENABLE_BF16
978
-////////////////////////////////////////////////////////////////////////////////////////////////////
979
-
980
-inline __device__ float sum(float v)
981
-{
982
- return v;
983
-}
984
-
985
-////////////////////////////////////////////////////////////////////////////////////////////////////
986
-
987
-inline __device__ float sum(float2 v)
988
-{
989
- return v.x + v.y;
990
-}
991
-
992
-////////////////////////////////////////////////////////////////////////////////////////////////////
993
-
994
-inline __device__ float sum(float4 v)
995
-{
996
- return v.x + v.y + v.z + v.w;
997
-}
998
-
999
-////////////////////////////////////////////////////////////////////////////////////////////////////
1000
-
1001
-#ifdef ENABLE_BF16
1002
-inline __device__ float sum(__nv_bfloat162 v)
1003
-{
1004
- float2 vf = bf1622float2(v);
1005
- return vf.x + vf.y;
1006
-}
1007
-
1008
-////////////////////////////////////////////////////////////////////////////////////////////////////
1009
-
1010
-inline __device__ float sum(bf16_4_t v)
1011
-{
1012
- return sum(v.x) + sum(v.y);
1013
-}
1014
-
1015
-////////////////////////////////////////////////////////////////////////////////////////////////////
1016
-
1017
-inline __device__ float sum(bf16_8_t v)
1018
-{
1019
- return sum(v.x) + sum(v.y) + sum(v.z) + sum(v.w);
1020
-}
1021
-#endif // ENABLE_BF16
1022
-////////////////////////////////////////////////////////////////////////////////////////////////////
1023
-
1024
-inline __device__ float sum(uint16_t v)
1025
-{
1026
- return half_to_float(v);
1027
-}
1028
-
1029
-////////////////////////////////////////////////////////////////////////////////////////////////////
1030
-
1031
-inline __device__ float sum(uint32_t v)
1032
-{
1033
- float2 tmp = half2_to_float2(v);
1034
- return tmp.x + tmp.y;
1035
-}
1036
-
1037
-////////////////////////////////////////////////////////////////////////////////////////////////////
1038
-
1039
-inline __device__ float sum(uint2 v)
1040
-{
1041
- uint32_t c = add(v.x, v.y);
1042
- return sum(c);
1043
-}
1044
-
1045
-////////////////////////////////////////////////////////////////////////////////////////////////////
1046
-
1047
-inline __device__ float sum(uint4 v)
1048
-{
1049
-#if 1
1050
- uint32_t c = add(v.x, v.y);
1051
- c = add(c, v.z);
1052
- c = add(c, v.w);
1053
-#else
1054
- uint32_t c = add(v.x, v.y);
1055
- uint32_t d = add(v.z, v.w);
1056
- c = add(c, d);
1057
-#endif
1058
- return sum(c);
1059
-}
1060
-
1061
-////////////////////////////////////////////////////////////////////////////////////////////////////
1062
-
1063
-inline __device__ float sum(Float4_ v)
1064
-{
1065
- return v.x.x + v.x.y + v.y.x + v.y.y;
1066
-}
1067
-
1068
-////////////////////////////////////////////////////////////////////////////////////////////////////
1069
-
1070
-inline __device__ float sum(Float8_ v)
1071
-{
1072
- return v.x.x + v.x.y + v.y.x + v.y.y + v.z.x + v.z.y + v.w.x + v.w.y;
1073
-}
1074
-
1075
-////////////////////////////////////////////////////////////////////////////////////////////////////
1076
-
1077
-inline __device__ float dot(float a, float b)
1078
-{
1079
- return a * b;
1080
-}
1081
-
1082
-////////////////////////////////////////////////////////////////////////////////////////////////////
1083
-
1084
-inline __device__ float dot(float2 a, float2 b)
1085
-{
1086
- float2 c = mul<float2, float2, float2>(a, b);
1087
- return c.x + c.y;
1088
-}
1089
-
1090
-////////////////////////////////////////////////////////////////////////////////////////////////////
1091
-
1092
-inline __device__ float dot(Float4_ a, Float4_ b)
1093
-{
1094
- float2 acc = mul<float2, float2, float2>(a.x, b.x);
1095
- acc = fma(a.y, b.y, acc);
1096
- return acc.x + acc.y;
1097
-}
1098
-
1099
-////////////////////////////////////////////////////////////////////////////////////////////////////
1100
-
1101
-inline __device__ float dot(Float8_ a, Float8_ b)
1102
-{
1103
- float2 acc = mul<float2, float2, float2>(a.x, b.x);
1104
- acc = fma(a.y, b.y, acc);
1105
- acc = fma(a.z, b.z, acc);
1106
- acc = fma(a.w, b.w, acc);
1107
- return acc.x + acc.y;
1108
-}
1109
-
1110
-////////////////////////////////////////////////////////////////////////////////////////////////////
1111
-
1112
-template<typename T>
1113
-inline __device__ float dot(T a, T b)
1114
-{
1115
- return sum(mul<T, T, T>(a, b));
1116
-}
1117
-
1118
-////////////////////////////////////////////////////////////////////////////////////////////////////
1119
-
1120
-template<typename A, typename T>
1121
-inline __device__ float dot(T a, T b)
1122
-{
1123
- return sum(mul<A, T, T>(a, b));
1124
-}
1125
-
1126
-////////////////////////////////////////////////////////////////////////////////////////////////////
1127
-
1128
-inline __device__ void zero(uint16_t& dst)
1129
-{
1130
- dst = uint16_t(0);
1131
-}
1132
-
1133
-////////////////////////////////////////////////////////////////////////////////////////////////////
1134
-
1135
-template<typename T>
1136
-inline __device__ void zero(T& dst)
1137
-{
1138
- constexpr int WORDS = sizeof(T) / 4;
1139
- union {
1140
- T raw;
1141
- uint32_t words[WORDS];
1142
- } tmp;
1143
-#pragma unroll
1144
- for (int ii = 0; ii < WORDS; ++ii) {
1145
- tmp.words[ii] = 0u;
1146
- }
1147
- dst = tmp.raw;
1148
-}
1149
-
1150
-
1151
-////////////////////////////////////////////////////////////////////////////////////////////////////
1152
-
1153
-inline __device__ void convert_from_float(float& dst, float src)
1154
-{
1155
- dst = src;
1156
-}
1157
-
1158
-////////////////////////////////////////////////////////////////////////////////////////////////////
1159
-
1160
-inline __device__ void convert_from_float(uint16_t& dst, float src)
1161
-{
1162
- dst = float_to_half(src);
1163
-}
1164
-
1165
-////////////////////////////////////////////////////////////////////////////////////////////////////
1166
-
1167
-inline __device__ void convert_from_float(uint32_t& dst, float2 src)
1168
-{
1169
- dst = float2_to_half2(src);
1170
-}
1171
-
1172
-////////////////////////////////////////////////////////////////////////////////////////////////////
1173
-#ifdef ENABLE_BF16
1174
-inline __device__ void convert_from_float(__nv_bfloat16& dst, float src)
1175
-{
1176
- dst = __float2bfloat16(src);
1177
-}
1178
-
1179
-////////////////////////////////////////////////////////////////////////////////////////////////////
1180
-
1181
-inline __device__ void convert_from_float(__nv_bfloat162& dst, float2 src)
1182
-{
1183
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1184
- dst = __float22bfloat162_rn(src);
1185
-#else
1186
- dst = __floats2bfloat162_rn(src.x, src.y);
1187
-#endif
1188
-}
1189
-#endif // ENABLE_BF16
1190
-////////////////////////////////////////////////////////////////////////////////////////////////////
1191
-
1192
-inline __device__ void convert_from_float(uint2& dst, Float4_ src)
1193
-{
1194
- dst.x = float2_to_half2(src.x);
1195
- dst.y = float2_to_half2(src.y);
1196
-}
1197
-
1198
-////////////////////////////////////////////////////////////////////////////////////////////////////
1199
-
1200
-inline __device__ void convert_from_float(uint4& dst, Float8_ src)
1201
-{
1202
- dst.x = float2_to_half2(src.x);
1203
- dst.y = float2_to_half2(src.y);
1204
- dst.z = float2_to_half2(src.z);
1205
- dst.w = float2_to_half2(src.w);
1206
-}
1207
-
1208
-////////////////////////////////////////////////////////////////////////////////////////////////////
1209
-
1210
-#ifdef ENABLE_BF16
1211
-inline __device__ void convert_from_float(bf16_4_t& dst, Float4_ src)
1212
-{
1213
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1214
- dst.x = __float22bfloat162_rn(src.x);
1215
- dst.y = __float22bfloat162_rn(src.y);
1216
-#else
1217
- dst.x = __floats2bfloat162_rn(src.x.x, src.x.y);
1218
- dst.y = __floats2bfloat162_rn(src.y.x, src.y.y);
1219
-#endif
1220
-}
1221
-
1222
-////////////////////////////////////////////////////////////////////////////////////////////////////
1223
-
1224
-inline __device__ void convert_from_float(bf16_8_t& dst, Float8_ src)
1225
-{
1226
-#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1227
- dst.x = __float22bfloat162_rn(src.x);
1228
- dst.y = __float22bfloat162_rn(src.y);
1229
- dst.z = __float22bfloat162_rn(src.z);
1230
- dst.w = __float22bfloat162_rn(src.w);
1231
-#else
1232
- dst.x = __floats2bfloat162_rn(src.x.x, src.x.y);
1233
- dst.y = __floats2bfloat162_rn(src.y.x, src.y.y);
1234
- dst.z = __floats2bfloat162_rn(src.z.x, src.z.y);
1235
- dst.w = __floats2bfloat162_rn(src.w.x, src.w.y);
1236
-#endif
1237
-}
1238
-#endif // ENABLE_BF16
1239
-
1240
-////////////////////////////////////////////////////////////////////////////////////////////////////
1241
-
1242
-inline __device__ void convert_from_float(float2& dst, float2 src)
1243
-{
1244
- dst = src;
1245
-}
1246
-
1247
-////////////////////////////////////////////////////////////////////////////////////////////////////
1248
-
1249
-inline __device__ void convert_from_float(float4& dst, float4 src)
1250
-{
1251
- dst = src;
1252
-}
1253
-
1254
-////////////////////////////////////////////////////////////////////////////////////////////////////
1255
-
1256
-inline __device__ float convert_to_float(float4 u)
1257
-{
1258
- return u.x;
1259
-}
1260
-
1261
-////////////////////////////////////////////////////////////////////////////////////////////////////
1262
-
1263
-inline __device__ float convert_to_float(uint4 u)
1264
-{
1265
- float2 tmp = half2_to_float2(u.x);
1266
- return tmp.x;
1267
-}
1268
-
1269
-////////////////////////////////////////////////////////////////////////////////////////////////////
1270
-
1271
-// inline __device__ float cast_to_float(float u)
1272
-// {
1273
-// return u;
1274
-// }
1275
-
1276
-////////////////////////////////////////////////////////////////////////////////////////////////////
1277
-
1278
-// inline __device__ float2 cast_to_float(float2 u)
1279
-// {
1280
-// return u;
1281
-// }
1282
-
1283
-////////////////////////////////////////////////////////////////////////////////////////////////////
1284
-
1285
-// inline __device__ float4 cast_to_float(float4 u)
1286
-// {
1287
-// return u;
1288
-// }
1289
-
1290
-////////////////////////////////////////////////////////////////////////////////////////////////////
1291
-
1292
-// inline __device__ Float4_ cast_to_float(Float4_ u)
1293
-// {
1294
-// return u;
1295
-// }
1296
-
1297
-////////////////////////////////////////////////////////////////////////////////////////////////////
1298
-
1299
-// inline __device__ Float8_ cast_to_float(Float8_ u)
1300
-// {
1301
-// return u;
1302
-// }
1303
-
1304
-////////////////////////////////////////////////////////////////////////////////////////////////////
1305
-
1306
-inline __device__ float cast_to_float(uint16_t u)
1307
-{
1308
- return half_to_float(u);
1309
-}
1310
-
1311
-////////////////////////////////////////////////////////////////////////////////////////////////////
1312
-
1313
-inline __device__ float2 cast_to_float(uint32_t u)
1314
-{
1315
- return half2_to_float2(u);
1316
-}
1317
-
1318
-////////////////////////////////////////////////////////////////////////////////////////////////////
1319
-
1320
-inline __device__ Float4_ cast_to_float(uint2 u)
1321
-{
1322
- Float4_ tmp;
1323
- tmp.x = half2_to_float2(u.x);
1324
- tmp.y = half2_to_float2(u.y);
1325
- return tmp;
1326
-}
1327
-
1328
-////////////////////////////////////////////////////////////////////////////////////////////////////
1329
-
1330
-inline __device__ Float8_ cast_to_float(uint4 u)
1331
-{
1332
- Float8_ tmp;
1333
- tmp.x = half2_to_float2(u.x);
1334
- tmp.y = half2_to_float2(u.y);
1335
- tmp.z = half2_to_float2(u.z);
1336
- tmp.w = half2_to_float2(u.w);
1337
- return tmp;
1338
-}
1339
-
1340
-}
csrc/layernorm_kernels.cu
+1
-1
@@ -1,7 +1,7 @@
1
#include <torch/extension.h>
2
#include <ATen/cuda/CUDAContext.h>
3
4
-#include "reduction_utils.h"
4
+#include "reduction_utils.cuh"
5
6
namespace cacheflow {
7
csrc/reduction_utils.cuh
new
+34
@@ -0,0 +1,34 @@
1
+#pragma once
2
+
3
+namespace cacheflow {
4
+
5
+template<typename T>
6
+__inline__ __device__ T warpReduceSum(T val) {
7
+#pragma unroll
8
+ for (int mask = 16; mask > 0; mask >>= 1)
9
+ val += __shfl_xor_sync(0xffffffff, val, mask, 32);
10
+ return val;
11
+}
12
+
13
+/* Calculate the sum of all elements in a block */
14
+template<typename T>
15
+__inline__ __device__ T blockReduceSum(T val) {
16
+ static __shared__ T shared[32];
17
+ int lane = threadIdx.x & 0x1f;
18
+ int wid = threadIdx.x >> 5;
19
+
20
+ val = warpReduceSum<T>(val);
21
+
22
+ if (lane == 0)
23
+ shared[wid] = val;
24
+
25
+ __syncthreads();
26
+
27
+ // Modify from blockDim.x << 5 to blockDim.x / 32. to prevent
28
+ // blockDim.x is not divided by 32
29
+ val = (threadIdx.x < (blockDim.x / 32.f)) ? shared[lane] : (T)(0.0f);
30
+ val = warpReduceSum<T>(val);
31
+ return val;
32
+}
33
+
34
+} // namespace cacheflow
csrc/reduction_utils.h
deleted
-76
@@ -1,76 +0,0 @@
1
-#pragma once
2
-
3
-namespace cacheflow {
4
-
5
-template<int WARPS_PER_BLOCK, int WARP_SIZE = 32>
6
-inline __device__ float block_sum(float* red_smem, float sum)
7
-{
8
-
9
- // Decompose the thread index into warp / lane.
10
- int warp = threadIdx.x / WARP_SIZE;
11
- int lane = threadIdx.x % WARP_SIZE;
12
-
13
- // Compute the sum per warp.
14
-#pragma unroll
15
- for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
16
- sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
17
- }
18
-
19
- // Warp leaders store the data to shared memory.
20
- if (lane == 0) {
21
- red_smem[warp] = sum;
22
- }
23
-
24
- // Make sure the data is in shared memory.
25
- __syncthreads();
26
-
27
- // The warps compute the final sums.
28
- if (lane < WARPS_PER_BLOCK) {
29
- sum = red_smem[lane];
30
- }
31
-
32
-// Parallel reduction inside the warp.
33
-#pragma unroll
34
- for (int mask = WARPS_PER_BLOCK / 2; mask >= 1; mask /= 2) {
35
- sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
36
- }
37
-
38
- // Broadcast to other threads.
39
- return __shfl_sync(uint32_t(-1), sum, 0);
40
-}
41
-
42
-#define FINAL_MASK 0xffffffff
43
-
44
-template<typename T>
45
-__inline__ __device__ T warpReduceSum(T val)
46
-{
47
-#pragma unroll
48
- for (int mask = 16; mask > 0; mask >>= 1)
49
- val += __shfl_xor_sync(FINAL_MASK, val, mask, 32);
50
- return val;
51
-}
52
-
53
-/* Calculate the sum of all elements in a block */
54
-template<typename T>
55
-__inline__ __device__ T blockReduceSum(T val)
56
-{
57
- static __shared__ T shared[32];
58
- int lane = threadIdx.x & 0x1f;
59
- int wid = threadIdx.x >> 5;
60
-
61
- val = warpReduceSum<T>(val);
62
-
63
- if (lane == 0)
64
- shared[wid] = val;
65
-
66
- __syncthreads();
67
-
68
- // Modify from blockDim.x << 5 to blockDim.x / 32. to prevent
69
- // blockDim.x is not divided by 32
70
- val = (threadIdx.x < (blockDim.x / 32.f)) ? shared[lane] : (T)(0.0f);
71
- val = warpReduceSum<T>(val);
72
-
73
- return val;
74
-}
75
-
76
-} // namespace cacheflow
setup.py
+1
-1
@@ -18,7 +18,7 @@ ext_modules.append(cache_extension)
18
# Attention kernels.
19
attention_extension = cpp_extension.CUDAExtension(
20
name='cacheflow.attention_ops',
21
- sources=['csrc/attention.cpp', 'csrc/attention_kernels.cu'],
21
+ sources=['csrc/attention.cpp', 'csrc/attention/attention_kernels.cu'],
22
extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
23
)
24
ext_modules.append(attention_extension)
tests/kernels/attention.py
-90
@@ -271,78 +271,6 @@ def test_multi_query_kv_attention(
271
assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
272
273
274
-def test_multi_query_cached_kv_attention(
275
- num_queries: int,
276
- num_heads: int,
277
- head_size: int,
278
- block_size: int,
279
- num_blocks: int,
280
- dtype: torch.dtype,
281
-) -> None:
282
- query_lens = random.sample(range(1, MAX_SEQ_LEN), num_queries)
283
- cu_query_lens = [0]
284
- for query_len in query_lens:
285
- cu_query_lens.append(cu_query_lens[-1] + query_len)
286
- num_total_tokens = cu_query_lens[-1]
287
-
288
- qkv = torch.randn(
289
- num_total_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
290
- query, _, _ = qkv.unbind(dim=1)
291
- x = 16 // torch.tensor([], dtype=dtype).element_size()
292
- key_block_shape = (num_heads, head_size // x, block_size, x)
293
- key_cache = torch.randn(
294
- size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
295
- value_block_shape = (num_heads, head_size, block_size)
296
- value_cache = torch.randn(
297
- size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
298
-
299
- cu_query_lens = torch.tensor(cu_query_lens, dtype=torch.int, device='cuda')
300
- context_lens = [
301
- query_len + random.randint(0, MAX_SEQ_LEN - query_len)
302
- for query_len in query_lens
303
- ]
304
- max_context_len = max(context_lens)
305
- context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
306
-
307
- max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
308
- block_tables = []
309
- for _ in range(num_queries):
310
- block_table = [
311
- random.randint(0, num_blocks - 1)
312
- for _ in range(max_num_blocks_per_seq)
313
- ]
314
- block_tables.append(block_table)
315
- block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
316
-
317
- scale = float(1.0 / (head_size ** 0.5))
318
- output = torch.empty(
319
- num_total_tokens, num_heads, head_size, dtype=dtype, device='cuda')
320
-
321
- attention_ops.multi_query_cached_kv_attention(
322
- cu_query_lens,
323
- output,
324
- query,
325
- key_cache,
326
- value_cache,
327
- scale,
328
- block_tables,
329
- context_lens,
330
- block_size,
331
- max_context_len,
332
- )
333
-
334
- ref_output = ref_multi_query_cached_kv_attention(
335
- cu_query_lens,
336
- query,
337
- key_cache,
338
- value_cache,
339
- block_tables,
340
- context_lens,
341
- dtype,
342
- )
343
- assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
344
-
345
-
274
@torch.inference_mode()
275
def test_attention(seed: int) -> None:
276
# NOTE(woosuk): Even when the seed is fixed, there is a chance that
@@ -364,24 +292,6 @@ def test_attention(seed: int) -> None:
292
dtype=dtype,
293
)
294
367
- # NOTE(siyuan): Same as above. Re-run the test if it fails. Also
368
- # note that the test is also more likely to fail due to the much
369
- # larger amount of tokens in the input may increase the variance.
370
- for dtype in [torch.half, torch.float]:
371
- for block_size in [8, 16, 32]:
372
- for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
373
- print(f'Testing multi_query_cached_kv_attention with '
374
- f'dtype={dtype}, block_size={block_size}, '
375
- f'head_size={head_size}')
376
- test_multi_query_cached_kv_attention(
377
- num_queries=11,
378
- num_heads=3,
379
- head_size=head_size,
380
- block_size=block_size,
381
- num_blocks=1024,
382
- dtype=dtype,
383
- )
384
-
295
# NOTE(woosuk): FlashAttention does not support FP32.
296
for dtype in [torch.half]:
297
# NOTE(woosuk): FlashAttention does not support head_size > 128.