@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 436e523bf1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Refactor attention kernels (#53)

Woosuk Kwon committed May 3, 2023 at 13:40 UTC 436e523bf1616bea22a1df78d93d7522311dccc8
14 files changed +1253 -2569
csrc/attention/attention_dtypes.cuh new
+5
@@ -0,0 +1,5 @@
1 +#pragma once
2 +
3 +#include "attention_generic.cuh"
4 +#include "dtype_float16.cuh"
5 +#include "dtype_float32.cuh"
csrc/attention/attention_generic.cuh new
+47
@@ -0,0 +1,47 @@
1 +#pragma once
2 +
3 +#include <stdint.h>
4 +
5 +namespace cacheflow {
6 +
7 +// A vector type to store Q, K, V elements.
8 +template<typename T, int VEC_SIZE>
9 +struct Vec {};
10 +
11 +// A vector type to store FP32 accumulators.
12 +template<typename T>
13 +struct FloatVec {};
14 +
15 +// Template vector operations.
16 +template<typename Acc, typename A, typename B>
17 +inline __device__ Acc mul(A a, B b);
18 +
19 +template<typename T>
20 +inline __device__ float sum(T v);
21 +
22 +template<typename T>
23 +inline __device__ float dot(T a, T b) {
24 + return sum(mul<T, T, T>(a, b));
25 +}
26 +
27 +template<typename A, typename T>
28 +inline __device__ float dot(T a, T b) {
29 + return sum(mul<A, T, T>(a, b));
30 +}
31 +
32 +template<typename T>
33 +inline __device__ void zero(T& dst) {
34 + constexpr int WORDS = sizeof(T) / 4;
35 + union {
36 + T raw;
37 + uint32_t words[WORDS];
38 + } tmp;
39 +
40 +#pragma unroll
41 + for (int ii = 0; ii < WORDS; ++ii) {
42 + tmp.words[ii] = 0u;
43 + }
44 + dst = tmp.raw;
45 +}
46 +
47 +} // namespace cacheflow
csrc/attention/attention_kernels.cu new
+451
@@ -0,0 +1,451 @@
1 +#include <torch/extension.h>
2 +#include <ATen/cuda/CUDAContext.h>
3 +
4 +#include "attention_dtypes.cuh"
5 +#include "attention_utils.cuh"
6 +
7 +#include <algorithm>
8 +
9 +#define WARP_SIZE 32
10 +#define MAX(a, b) ((a) > (b) ? (a) : (b))
11 +#define MIN(a, b) ((a) < (b) ? (a) : (b))
12 +
13 +namespace cacheflow {
14 +
15 +// Utility function for attention softmax.
16 +template<int NUM_WARPS>
17 +inline __device__ float block_sum(float* red_smem, float sum) {
18 + // Decompose the thread index into warp / lane.
19 + int warp = threadIdx.x / WARP_SIZE;
20 + int lane = threadIdx.x % WARP_SIZE;
21 +
22 + // Compute the sum per warp.
23 +#pragma unroll
24 + for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
25 + sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
26 + }
27 +
28 + // Warp leaders store the data to shared memory.
29 + if (lane == 0) {
30 + red_smem[warp] = sum;
31 + }
32 +
33 + // Make sure the data is in shared memory.
34 + __syncthreads();
35 +
36 + // The warps compute the final sums.
37 + if (lane < NUM_WARPS) {
38 + sum = red_smem[lane];
39 + }
40 +
41 + // Parallel reduction inside the warp.
42 +#pragma unroll
43 + for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
44 + sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
45 + }
46 +
47 + // Broadcast to other threads.
48 + return __shfl_sync(uint32_t(-1), sum, 0);
49 +}
50 +
51 +// Grid: (num_heads, num_seqs).
52 +template<
53 + typename scalar_t,
54 + int HEAD_SIZE,
55 + int BLOCK_SIZE,
56 + int NUM_THREADS>
57 +__global__ void single_query_cached_kv_attention_kernel(
58 + scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
59 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
60 + const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
61 + const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
62 + const float scale,
63 + const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
64 + const int* __restrict__ context_lens, // [num_seqs]
65 + const int max_num_blocks_per_seq,
66 + const int q_stride) {
67 + constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
68 + constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
69 + constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
70 + const int thread_idx = threadIdx.x;
71 + const int warp_idx = thread_idx / WARP_SIZE;
72 + const int lane = thread_idx % WARP_SIZE;
73 +
74 + const int head_idx = blockIdx.x;
75 + const int num_heads = gridDim.x;
76 + const int seq_idx = blockIdx.y;
77 +
78 + // A vector type to store a part of a key or a query.
79 + // The vector size is configured in such a way that the threads in a thread group
80 + // fetch or compute 16 bytes at a time.
81 + // For example, if the size of a thread group is 4 and the data type is half,
82 + // then the vector size is 16 / (4 * sizeof(half)) == 2.
83 + constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
84 + using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
85 + using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
86 +
87 + constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
88 + constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
89 +
90 + const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
91 + const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
92 +
93 + // Load the query to registers.
94 + // Each thread in a thread group has a different part of the query.
95 + // For example, if the the thread group size is 4, then the first thread in the group
96 + // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
97 + // th vectors of the query, and so on.
98 + // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
99 + const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
100 + Q_vec q_vecs[NUM_VECS_PER_THREAD];
101 +#pragma unroll
102 + for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
103 + const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
104 + q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
105 + }
106 +
107 + // Memory planning.
108 + extern __shared__ char shared_mem[];
109 + // NOTE(woosuk): We use FP32 for the softmax logits for better accuracy.
110 + float* logits = reinterpret_cast<float*>(shared_mem);
111 + // Workspace for reduction.
112 + __shared__ float red_smem[2 * NUM_WARPS];
113 +
114 + // x == THREAD_GROUP_SIZE * VEC_SIZE
115 + // Each thread group fetches x elements from the key at a time.
116 + constexpr int x = 16 / sizeof(scalar_t);
117 + float qk_max = -FLT_MAX;
118 +
119 + const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
120 + const int context_len = context_lens[seq_idx];
121 + const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
122 +
123 + // Iterate over the key blocks.
124 + // Each warp fetches a block of keys for each iteration.
125 + // Each thread group in a warp fetches a key from the block, and computes
126 + // dot product with the query.
127 + for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
128 + const int physical_block_number = block_table[block_idx];
129 +
130 + // Load a key to registers.
131 + // Each thread in a thread group has a different part of the key.
132 + // For example, if the the thread group size is 4, then the first thread in the group
133 + // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
134 + // vectors of the key, and so on.
135 + for (int i = 0; i < NUM_TOKENS_PER_THREAD_GROUP; i++) {
136 + const int physical_block_offset = (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;
137 + const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
138 + K_vec k_vecs[NUM_VECS_PER_THREAD];
139 +
140 +#pragma unroll
141 + for (int j = 0; j < NUM_VECS_PER_THREAD; j++) {
142 + const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
143 + + head_idx * HEAD_SIZE * BLOCK_SIZE
144 + + physical_block_offset * x;
145 + const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
146 + const int offset1 = (vec_idx * VEC_SIZE) / x;
147 + const int offset2 = (vec_idx * VEC_SIZE) % x;
148 + k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
149 + }
150 +
151 + // Compute dot product.
152 + // This includes a reduction across the threads in the same thread group.
153 + const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
154 + const bool mask = token_idx >= context_len;
155 +
156 + if (thread_group_offset == 0) {
157 + // Store the partial reductions to shared memory.
158 + // NOTE(woosuk): It is required to zero out the masked logits.
159 + logits[token_idx] = mask ? 0.f : qk;
160 + // Update the max value.
161 + qk_max = mask ? qk_max : fmaxf(qk_max, qk);
162 + }
163 + }
164 + }
165 +
166 + // Perform reduction across the threads in the same warp to get the
167 + // max qk value for each "warp" (not across the thread block yet).
168 + // The 0-th thread of each thread group already has its max qk value.
169 +#pragma unroll
170 + for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
171 + qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
172 + }
173 + if (lane == 0) {
174 + red_smem[warp_idx] = qk_max;
175 + }
176 + __syncthreads();
177 +
178 + // TODO(woosuk): Refactor this part.
179 + // Get the max qk value for the sequence.
180 + qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
181 +#pragma unroll
182 + for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
183 + qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
184 + }
185 + // Broadcast the max qk value to all threads.
186 + qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
187 +
188 + // Get the sum of the exp values.
189 + float exp_sum = 0.f;
190 + for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
191 + float val = __expf(logits[i] - qk_max);
192 + logits[i] = val;
193 + exp_sum += val;
194 + }
195 + exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
196 +
197 + // Compute softmax.
198 + const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
199 + for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
200 + logits[i] *= inv_sum;
201 + }
202 + __syncthreads();
203 +
204 + // Each thread will fetch 16 bytes from the value cache at a time.
205 + constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
206 + using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
207 + using L_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
208 + using Float_L_vec = typename FloatVec<L_vec>::Type;
209 +
210 + constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
211 + constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
212 + constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
213 +
214 + // NOTE(woosuk): We use FP32 for the accumulator for better accuracy.
215 + float accs[NUM_ROWS_PER_THREAD];
216 +#pragma unroll
217 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
218 + accs[i] = 0.f;
219 + }
220 +
221 + for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
222 + const int physical_block_number = block_table[block_idx];
223 + const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
224 + const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
225 + L_vec logits_vec;
226 + from_float(logits_vec, *reinterpret_cast<Float_L_vec*>(logits + token_idx));
227 +
228 + const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
229 + + head_idx * HEAD_SIZE * BLOCK_SIZE;
230 +#pragma unroll
231 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
232 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
233 + if (row_idx < HEAD_SIZE) {
234 + const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
235 + V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
236 + accs[i] += dot(logits_vec, v_vec);
237 + }
238 + }
239 + }
240 +
241 + // Perform reduction within each warp.
242 +#pragma unroll
243 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
244 + float acc = accs[i];
245 +#pragma unroll
246 + for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
247 + acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
248 + }
249 + accs[i] = acc;
250 + }
251 +
252 + // NOTE(woosuk): A barrier is required because the shared memory space for logits
253 + // is reused for the output.
254 + __syncthreads();
255 +
256 + // Perform reduction across warps.
257 + float* out_smem = reinterpret_cast<float*>(shared_mem);
258 +#pragma unroll
259 + for (int i = NUM_WARPS; i > 1; i /= 2) {
260 + int mid = i / 2;
261 + // Upper warps write to shared memory.
262 + if (warp_idx >= mid && warp_idx < i) {
263 + float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
264 +#pragma unroll
265 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
266 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
267 + if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
268 + dst[row_idx] = accs[i];
269 + }
270 + }
271 + }
272 + __syncthreads();
273 +
274 + // Lower warps update the output.
275 + if (warp_idx < mid) {
276 + const float* src = &out_smem[warp_idx * HEAD_SIZE];
277 +#pragma unroll
278 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
279 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
280 + if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
281 + accs[i] += src[row_idx];
282 + }
283 + }
284 + }
285 + __syncthreads();
286 + }
287 +
288 + // Write the final output.
289 + if (warp_idx == 0) {
290 + scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
291 +#pragma unroll
292 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
293 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
294 + if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
295 + from_float(*(out_ptr + row_idx), accs[i]);
296 + }
297 + }
298 + }
299 +}
300 +
301 +} // namespace cacheflow
302 +
303 +#define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
304 + cacheflow::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
305 + <<<grid, block, shared_mem_size, stream>>>( \
306 + out_ptr, \
307 + query_ptr, \
308 + key_cache_ptr, \
309 + value_cache_ptr, \
310 + scale, \
311 + block_tables_ptr, \
312 + context_lens_ptr, \
313 + max_num_blocks_per_seq, \
314 + query_stride);
315 +
316 +// TODO(woosuk): Tune NUM_THREADS.
317 +template<
318 + typename T,
319 + int BLOCK_SIZE,
320 + int NUM_THREADS = 128>
321 +void single_query_cached_kv_attention_launcher(
322 + torch::Tensor& out,
323 + torch::Tensor& query,
324 + torch::Tensor& key_cache,
325 + torch::Tensor& value_cache,
326 + float scale,
327 + torch::Tensor& block_tables,
328 + torch::Tensor& context_lens,
329 + int max_context_len) {
330 + int num_seqs = query.size(0);
331 + int num_heads = query.size(1);
332 + int head_size = query.size(2);
333 + int max_num_blocks_per_seq = block_tables.size(1);
334 + int query_stride = query.stride(0);
335 +
336 + int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
337 + assert(head_size % thread_group_size == 0);
338 +
339 + T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
340 + T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
341 + T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
342 + T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
343 + int* block_tables_ptr = block_tables.data_ptr<int>();
344 + int* context_lens_ptr = context_lens.data_ptr<int>();
345 +
346 + constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
347 + int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
348 + int logits_size = padded_max_context_len * sizeof(T);
349 + int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
350 + int shared_mem_size = std::max(logits_size, outputs_size);
351 +
352 + dim3 grid(num_heads, num_seqs);
353 + dim3 block(NUM_THREADS);
354 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
355 + switch (head_size) {
356 + case 32:
357 + LAUNCH_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
358 + break;
359 + case 64:
360 + LAUNCH_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
361 + break;
362 + case 80:
363 + LAUNCH_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
364 + break;
365 + case 96:
366 + LAUNCH_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
367 + break;
368 + case 128:
369 + LAUNCH_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
370 + break;
371 + case 160:
372 + LAUNCH_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
373 + break;
374 + case 192:
375 + LAUNCH_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
376 + break;
377 + case 256:
378 + LAUNCH_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
379 + break;
380 + default:
381 + TORCH_CHECK(false, "Unsupported head size: ", head_size);
382 + break;
383 + }
384 +}
385 +
386 +#define CALL_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
387 + single_query_cached_kv_attention_launcher<T, BLOCK_SIZE>( \
388 + out, \
389 + query, \
390 + key_cache, \
391 + value_cache, \
392 + scale, \
393 + block_tables, \
394 + context_lens, \
395 + max_context_len);
396 +
397 +#define CALL_KERNEL_LAUNCHER_BLOCK_SIZE(T) \
398 + switch (block_size) { \
399 + case 1: \
400 + CALL_KERNEL_LAUNCHER(T, 1); \
401 + break; \
402 + case 2: \
403 + CALL_KERNEL_LAUNCHER(T, 2); \
404 + break; \
405 + case 4: \
406 + CALL_KERNEL_LAUNCHER(T, 4); \
407 + break; \
408 + case 8: \
409 + CALL_KERNEL_LAUNCHER(T, 8); \
410 + break; \
411 + case 16: \
412 + CALL_KERNEL_LAUNCHER(T, 16); \
413 + break; \
414 + case 32: \
415 + CALL_KERNEL_LAUNCHER(T, 32); \
416 + break; \
417 + case 64: \
418 + CALL_KERNEL_LAUNCHER(T, 64); \
419 + break; \
420 + case 128: \
421 + CALL_KERNEL_LAUNCHER(T, 128); \
422 + break; \
423 + case 256: \
424 + CALL_KERNEL_LAUNCHER(T, 256); \
425 + break; \
426 + default: \
427 + TORCH_CHECK(false, "Unsupported block size: ", block_size); \
428 + break; \
429 + }
430 +
431 +void single_query_cached_kv_attention(
432 + torch::Tensor& out, // [num_seqs, num_heads, head_size]
433 + torch::Tensor& query, // [num_seqs, num_heads, head_size]
434 + torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
435 + torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
436 + float scale,
437 + torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
438 + torch::Tensor& context_lens, // [num_seqs]
439 + int block_size,
440 + int max_context_len) {
441 + // TODO(woosuk): Support FP32 and BF16.
442 + if (query.dtype() == at::ScalarType::Half) {
443 + CALL_KERNEL_LAUNCHER_BLOCK_SIZE(uint16_t);
444 + } else {
445 + TORCH_CHECK(false, "Unsupported data type: ", query.dtype());
446 + }
447 +}
448 +
449 +#undef WARP_SIZE
450 +#undef MAX
451 +#undef MIN
csrc/attention/attention_utils.cuh new
+38
@@ -0,0 +1,38 @@
1 +#pragma once
2 +
3 +#include "attention_dtypes.cuh"
4 +
5 +#include <float.h>
6 +#include <type_traits>
7 +
8 +namespace cacheflow {
9 +
10 +// Q*K^T operation.
11 +template<int THREAD_GROUP_SIZE, typename Vec, int N>
12 +inline __device__ float qk_dot_(const Vec (&q)[N], const Vec (&k)[N]) {
13 + using A_vec = typename FloatVec<Vec>::Type;
14 + // Compute the parallel products for Q*K^T (treat vector lanes separately).
15 + A_vec qk_vec = mul<A_vec, Vec, Vec>(q[0], k[0]);
16 +#pragma unroll
17 + for (int ii = 1; ii < N; ++ii) {
18 + qk_vec = fma(q[ii], k[ii], qk_vec);
19 + }
20 +
21 + // Finalize the reduction across lanes.
22 + float qk = sum(qk_vec);
23 +#pragma unroll
24 + for (int mask = THREAD_GROUP_SIZE / 2; mask >= 1; mask /= 2) {
25 + qk += __shfl_xor_sync(uint32_t(-1), qk, mask);
26 + }
27 + return qk;
28 +}
29 +
30 +template<typename T, int THREAD_GROUP_SIZE>
31 +struct Qk_dot {
32 + template<typename Vec, int N>
33 + static inline __device__ float dot(const Vec (&q)[N], const Vec (&k)[N]) {
34 + return qk_dot_<THREAD_GROUP_SIZE>(q, k);
35 + }
36 +};
37 +
38 +} // namespace cacheflow
csrc/attention/dtype_float16.cuh new
+426
@@ -0,0 +1,426 @@
1 +#pragma once
2 +
3 +#include "attention_generic.cuh"
4 +#include "dtype_float32.cuh"
5 +
6 +#include <stdint.h>
7 +
8 +namespace cacheflow {
9 +
10 +// FP16 vector types for Q, K, V.
11 +template<>
12 +struct Vec<uint16_t, 1> {
13 + using Type = uint16_t;
14 +};
15 +template<>
16 +struct Vec<uint16_t, 2> {
17 + using Type = uint32_t;
18 +};
19 +template<>
20 +struct Vec<uint16_t, 4> {
21 + using Type = uint2;
22 +};
23 +template<>
24 +struct Vec<uint16_t, 8> {
25 + using Type = uint4;
26 +};
27 +
28 +// FP32 accumulator vector types corresponding to Vec.
29 +template<>
30 +struct FloatVec<uint16_t> {
31 + using Type = float;
32 +};
33 +template<>
34 +struct FloatVec<uint32_t> {
35 + using Type = float2;
36 +};
37 +template<>
38 +struct FloatVec<uint2> {
39 + using Type = Float4_;
40 +};
41 +template<>
42 +struct FloatVec<uint4> {
43 + using Type = Float8_;
44 +};
45 +
46 +// Utility functions for type conversions.
47 +inline __device__ uint32_t h0_h0(uint16_t a) {
48 + uint32_t b;
49 + asm volatile("mov.b32 %0, {%1, %1};" : "=r"(b) : "h"(a));
50 + return b;
51 +}
52 +
53 +inline __device__ float half_to_float(uint16_t h) {
54 + float f;
55 + asm volatile("cvt.f32.f16 %0, %1;\n" : "=f"(f) : "h"(h));
56 + return f;
57 +}
58 +
59 +inline __device__ float2 half2_to_float2(uint32_t v) {
60 + uint16_t lo, hi;
61 + asm volatile("mov.b32 {%0, %1}, %2;\n" : "=h"(lo), "=h"(hi) : "r"(v));
62 + return make_float2(half_to_float(lo), half_to_float(hi));
63 +}
64 +
65 +inline __device__ uint16_t float_to_half(float f) {
66 + union {
67 + uint32_t u32;
68 + uint16_t u16[2];
69 + } tmp;
70 + asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f));
71 + return tmp.u16[0];
72 +}
73 +
74 +inline __device__ uint32_t float2_to_half2(float2 f) {
75 + union {
76 + uint32_t u32;
77 + uint16_t u16[2];
78 + } tmp;
79 +
80 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
81 + asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(f.y), "f"(f.x));
82 +#else
83 + asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
84 + asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
85 +#endif
86 + return tmp.u32;
87 +}
88 +
89 +// Vector addition.
90 +inline __device__ uint16_t add(uint16_t a, uint16_t b) {
91 + uint16_t c;
92 + asm volatile("add.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
93 + return c;
94 +}
95 +
96 +inline __device__ uint32_t add(uint32_t a, uint32_t b) {
97 + uint32_t c;
98 + asm volatile("add.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
99 + return c;
100 +}
101 +
102 +inline __device__ uint2 add(uint2 a, uint2 b) {
103 + uint2 c;
104 + c.x = add(a.x, b.x);
105 + c.y = add(a.y, b.y);
106 + return c;
107 +}
108 +
109 +inline __device__ uint4 add(uint4 a, uint4 b) {
110 + uint4 c;
111 + c.x = add(a.x, b.x);
112 + c.y = add(a.y, b.y);
113 + c.z = add(a.z, b.z);
114 + c.w = add(a.w, b.w);
115 + return c;
116 +}
117 +
118 +inline __device__ float2 add(uint32_t a, float2 fb) {
119 + float2 fa = half2_to_float2(a);
120 + return add(fa, fb);
121 +}
122 +
123 +inline __device__ Float4_ add(uint2 a, Float4_ fb) {
124 + Float4_ fc;
125 + fc.x = add(a.x, fb.x);
126 + fc.y = add(a.y, fb.y);
127 + return fc;
128 +}
129 +
130 +inline __device__ Float8_ add(uint4 a, Float8_ fb) {
131 + Float8_ fc;
132 + fc.x = add(a.x, fb.x);
133 + fc.y = add(a.y, fb.y);
134 + fc.z = add(a.z, fb.z);
135 + fc.w = add(a.w, fb.w);
136 + return fc;
137 +}
138 +
139 +// Vector multiplication.
140 +template<>
141 +inline __device__ uint16_t mul(uint16_t a, uint16_t b) {
142 + uint16_t c;
143 + asm volatile("mul.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
144 + return c;
145 +}
146 +
147 +template<>
148 +inline __device__ uint32_t mul(uint32_t a, uint32_t b) {
149 + uint32_t c;
150 + asm volatile("mul.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
151 + return c;
152 +}
153 +
154 +template<>
155 +inline __device__ uint32_t mul(uint16_t a, uint32_t b) {
156 + return mul<uint32_t, uint32_t, uint32_t>(h0_h0(a), b);
157 +}
158 +
159 +template<>
160 +inline __device__ uint2 mul(uint2 a, uint2 b) {
161 + uint2 c;
162 + c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
163 + c.y = mul<uint32_t, uint32_t, uint32_t>(a.y, b.y);
164 + return c;
165 +}
166 +
167 +template<>
168 +inline __device__ uint2 mul(uint16_t a, uint2 b) {
169 + uint32_t s = h0_h0(a);
170 + uint2 c;
171 + c.x = mul<uint32_t, uint32_t, uint32_t>(s, b.x);
172 + c.y = mul<uint32_t, uint32_t, uint32_t>(s, b.y);
173 + return c;
174 +}
175 +
176 +template<>
177 +inline __device__ uint4 mul(uint4 a, uint4 b) {
178 + uint4 c;
179 + c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
180 + c.y = mul<uint32_t, uint32_t, uint32_t>(a.y, b.y);
181 + c.z = mul<uint32_t, uint32_t, uint32_t>(a.z, b.z);
182 + c.w = mul<uint32_t, uint32_t, uint32_t>(a.w, b.w);
183 + return c;
184 +}
185 +
186 +template<>
187 +inline __device__ uint4 mul(uint16_t a, uint4 b) {
188 + uint32_t s = h0_h0(a);
189 + uint4 c;
190 + c.x = mul<uint32_t, uint32_t, uint32_t>(s, b.x);
191 + c.y = mul<uint32_t, uint32_t, uint32_t>(s, b.y);
192 + c.z = mul<uint32_t, uint32_t, uint32_t>(s, b.z);
193 + c.w = mul<uint32_t, uint32_t, uint32_t>(s, b.w);
194 + return c;
195 +}
196 +
197 +template<>
198 +inline __device__ float mul(uint16_t a, uint16_t b) {
199 + float fa = half_to_float(a);
200 + float fb = half_to_float(b);
201 + return fa * fb;
202 +}
203 +
204 +template<>
205 +inline __device__ float2 mul(uint32_t a, uint32_t b) {
206 + float2 fa = half2_to_float2(a);
207 + float2 fb = half2_to_float2(b);
208 + return mul<float2, float2, float2>(fa, fb);
209 +}
210 +
211 +template<>
212 +inline __device__ float2 mul(uint16_t a, uint32_t b) {
213 + return mul<float2, uint32_t, uint32_t>(h0_h0(a), b);
214 +}
215 +
216 +template<>
217 +inline __device__ Float4_ mul(uint2 a, uint2 b) {
218 + Float4_ fc;
219 + fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
220 + fc.y = mul<float2, uint32_t, uint32_t>(a.y, b.y);
221 + return fc;
222 +}
223 +
224 +template<>
225 +inline __device__ Float4_ mul(uint16_t a, uint2 b) {
226 + uint32_t s = h0_h0(a);
227 + Float4_ fc;
228 + fc.x = mul<float2, uint32_t, uint32_t>(s, b.x);
229 + fc.y = mul<float2, uint32_t, uint32_t>(s, b.y);
230 + return fc;
231 +}
232 +
233 +template<>
234 +inline __device__ Float8_ mul(uint4 a, uint4 b) {
235 + Float8_ fc;
236 + fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
237 + fc.y = mul<float2, uint32_t, uint32_t>(a.y, b.y);
238 + fc.z = mul<float2, uint32_t, uint32_t>(a.z, b.z);
239 + fc.w = mul<float2, uint32_t, uint32_t>(a.w, b.w);
240 + return fc;
241 +}
242 +
243 +template<>
244 +inline __device__ Float8_ mul(uint16_t a, uint4 b) {
245 + uint32_t s = h0_h0(a);
246 + Float8_ fc;
247 + fc.x = mul<float2, uint32_t, uint32_t>(s, b.x);
248 + fc.y = mul<float2, uint32_t, uint32_t>(s, b.y);
249 + fc.z = mul<float2, uint32_t, uint32_t>(s, b.z);
250 + fc.w = mul<float2, uint32_t, uint32_t>(s, b.w);
251 + return fc;
252 +}
253 +
254 +// Vector fused multiply-add.
255 +inline __device__ uint32_t fma(uint32_t a, uint32_t b, uint32_t c) {
256 + uint32_t d;
257 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(d) : "r"(a), "r"(b), "r"(c));
258 + return d;
259 +}
260 +
261 +inline __device__ uint32_t fma(uint16_t a, uint32_t b, uint32_t c) {
262 + return fma(h0_h0(a), b, c);
263 +}
264 +
265 +inline __device__ uint2 fma(uint2 a, uint2 b, uint2 c) {
266 + uint2 d;
267 + d.x = fma(a.x, b.x, c.x);
268 + d.y = fma(a.y, b.y, c.y);
269 + return d;
270 +}
271 +
272 +inline __device__ uint2 fma(uint16_t a, uint2 b, uint2 c) {
273 + uint32_t s = h0_h0(a);
274 + uint2 d;
275 + d.x = fma(s, b.x, c.x);
276 + d.y = fma(s, b.y, c.y);
277 + return d;
278 +}
279 +
280 +inline __device__ uint4 fma(uint4 a, uint4 b, uint4 c) {
281 + uint4 d;
282 + d.x = fma(a.x, b.x, c.x);
283 + d.y = fma(a.y, b.y, c.y);
284 + d.z = fma(a.z, b.z, c.z);
285 + d.w = fma(a.w, b.w, c.w);
286 + return d;
287 +}
288 +
289 +inline __device__ uint4 fma(uint16_t a, uint4 b, uint4 c) {
290 + uint32_t s = h0_h0(a);
291 + uint4 d;
292 + d.x = fma(s, b.x, c.x);
293 + d.y = fma(s, b.y, c.y);
294 + d.z = fma(s, b.z, c.z);
295 + d.w = fma(s, b.w, c.w);
296 + return d;
297 +}
298 +
299 +inline __device__ float fma(uint16_t a, uint16_t b, float fc) {
300 + float fa = half_to_float(a);
301 + float fb = half_to_float(b);
302 + return fa * fb + fc;
303 +}
304 +
305 +inline __device__ float2 fma(uint32_t a, uint32_t b, float2 fc) {
306 + float2 fa = half2_to_float2(a);
307 + float2 fb = half2_to_float2(b);
308 + return fma(fa, fb, fc);
309 +}
310 +
311 +inline __device__ float2 fma(uint16_t a, uint32_t b, float2 fc) {
312 + return fma(h0_h0(a), b, fc);
313 +}
314 +
315 +inline __device__ Float4_ fma(uint2 a, uint2 b, Float4_ fc) {
316 + Float4_ fd;
317 + fd.x = fma(a.x, b.x, fc.x);
318 + fd.y = fma(a.y, b.y, fc.y);
319 + return fd;
320 +}
321 +
322 +inline __device__ Float4_ fma(uint16_t a, uint2 b, Float4_ fc) {
323 + uint32_t s = h0_h0(a);
324 + Float4_ fd;
325 + fd.x = fma(s, b.x, fc.x);
326 + fd.y = fma(s, b.y, fc.y);
327 + return fd;
328 +}
329 +
330 +inline __device__ Float8_ fma(uint4 a, uint4 b, Float8_ fc) {
331 + Float8_ fd;
332 + fd.x = fma(a.x, b.x, fc.x);
333 + fd.y = fma(a.y, b.y, fc.y);
334 + fd.z = fma(a.z, b.z, fc.z);
335 + fd.w = fma(a.w, b.w, fc.w);
336 + return fd;
337 +}
338 +
339 +inline __device__ Float8_ fma(uint16_t a, uint4 b, Float8_ fc) {
340 + uint32_t s = h0_h0(a);
341 + Float8_ fd;
342 + fd.x = fma(s, b.x, fc.x);
343 + fd.y = fma(s, b.y, fc.y);
344 + fd.z = fma(s, b.z, fc.z);
345 + fd.w = fma(s, b.w, fc.w);
346 + return fd;
347 +}
348 +
349 +// Vector sum.
350 +template<>
351 +inline __device__ float sum(uint16_t v) {
352 + return half_to_float(v);
353 +}
354 +
355 +template<>
356 +inline __device__ float sum(uint32_t v) {
357 + float2 tmp = half2_to_float2(v);
358 + return tmp.x + tmp.y;
359 +}
360 +
361 +template<>
362 +inline __device__ float sum(uint2 v) {
363 + uint32_t c = add(v.x, v.y);
364 + return sum(c);
365 +}
366 +
367 +template<>
368 +inline __device__ float sum(uint4 v) {
369 + uint32_t c = add(v.x, v.y);
370 + c = add(c, v.z);
371 + c = add(c, v.w);
372 + return sum(c);
373 +}
374 +
375 +// Zero-out a vector.
376 +inline __device__ void zero(uint16_t& dst) {
377 + dst = uint16_t(0);
378 +}
379 +
380 +// From float32 to float16.
381 +inline __device__ void from_float(uint16_t& dst, float src) {
382 + dst = float_to_half(src);
383 +}
384 +
385 +inline __device__ void from_float(uint32_t& dst, float2 src) {
386 + dst = float2_to_half2(src);
387 +}
388 +
389 +inline __device__ void from_float(uint2& dst, Float4_ src) {
390 + dst.x = float2_to_half2(src.x);
391 + dst.y = float2_to_half2(src.y);
392 +}
393 +
394 +inline __device__ void from_float(uint4& dst, Float8_ src) {
395 + dst.x = float2_to_half2(src.x);
396 + dst.y = float2_to_half2(src.y);
397 + dst.z = float2_to_half2(src.z);
398 + dst.w = float2_to_half2(src.w);
399 +}
400 +
401 +// From float16 to float32.
402 +inline __device__ float to_float(uint16_t u) {
403 + return half_to_float(u);
404 +}
405 +
406 +inline __device__ float2 to_float(uint32_t u) {
407 + return half2_to_float2(u);
408 +}
409 +
410 +inline __device__ Float4_ to_float(uint2 u) {
411 + Float4_ tmp;
412 + tmp.x = half2_to_float2(u.x);
413 + tmp.y = half2_to_float2(u.y);
414 + return tmp;
415 +}
416 +
417 +inline __device__ Float8_ to_float(uint4 u) {
418 + Float8_ tmp;
419 + tmp.x = half2_to_float2(u.x);
420 + tmp.y = half2_to_float2(u.y);
421 + tmp.z = half2_to_float2(u.z);
422 + tmp.w = half2_to_float2(u.w);
423 + return tmp;
424 +}
425 +
426 +} // namespace cacheflow
csrc/attention/dtype_float32.cuh new
+250
@@ -0,0 +1,250 @@
1 +#pragma once
2 +
3 +#include "attention_generic.cuh"
4 +
5 +#include <stdint.h>
6 +
7 +namespace cacheflow {
8 +
9 +// Define FP32 vector data types.
10 +struct Float4_ {
11 + float2 x;
12 + float2 y;
13 +};
14 +
15 +struct Float8_ {
16 + float2 x;
17 + float2 y;
18 + float2 z;
19 + float2 w;
20 +};
21 +
22 +// FP32 vector types for Q, K, V.
23 +template<>
24 +struct Vec<float, 1> {
25 + using Type = float;
26 +};
27 +template<>
28 +struct Vec<float, 2> {
29 + using Type = float2;
30 +};
31 +template<>
32 +struct Vec<float, 4> {
33 + using Type = float4;
34 +};
35 +
36 +// FP32 accumulator vector types corresponding to Vec.
37 +template<>
38 +struct FloatVec<float> {
39 + using Type = float;
40 +};
41 +template<>
42 +struct FloatVec<float2> {
43 + using Type = float2;
44 +};
45 +template<>
46 +struct FloatVec<float4> {
47 + using Type = float4;
48 +};
49 +
50 +// Vector addition.
51 +inline __device__ float add(float a, float b) {
52 + return a + b;
53 +}
54 +
55 +inline __device__ float2 add(float2 a, float2 b) {
56 + float2 c;
57 + c.x = add(a.x, b.x);
58 + c.y = add(a.y, b.y);
59 + return c;
60 +}
61 +
62 +inline __device__ float4 add(float4 a, float4 b) {
63 + float4 c;
64 + c.x = add(a.x, b.x);
65 + c.y = add(a.y, b.y);
66 + c.z = add(a.z, b.z);
67 + c.w = add(a.w, b.w);
68 + return c;
69 +}
70 +
71 +// Vector multiplication.
72 +template<>
73 +inline __device__ float mul<float, float>(float a, float b) {
74 + return a * b;
75 +}
76 +
77 +template<>
78 +inline __device__ float2 mul(float2 a, float2 b) {
79 + float2 c;
80 + c.x = a.x * b.x;
81 + c.y = a.y * b.y;
82 + return c;
83 +}
84 +
85 +template<>
86 +inline __device__ float2 mul(float a, float2 b) {
87 + float2 c;
88 + c.x = a * b.x;
89 + c.y = a * b.y;
90 + return c;
91 +}
92 +
93 +template<>
94 +inline __device__ float4 mul(float4 a, float4 b) {
95 + float4 c;
96 + c.x = a.x * b.x;
97 + c.y = a.y * b.y;
98 + c.z = a.z * b.z;
99 + c.w = a.w * b.w;
100 + return c;
101 +}
102 +
103 +template<>
104 +inline __device__ float4 mul(float a, float4 b) {
105 + float4 c;
106 + c.x = a * b.x;
107 + c.y = a * b.y;
108 + c.z = a * b.z;
109 + c.w = a * b.w;
110 + return c;
111 +}
112 +
113 +// Vector fused multiply-add.
114 +inline __device__ float fma(float a, float b, float c) {
115 + return a * b + c;
116 +}
117 +
118 +inline __device__ float2 fma(float2 a, float2 b, float2 c) {
119 + float2 d;
120 + d.x = fma(a.x, b.x, c.x);
121 + d.y = fma(a.y, b.y, c.y);
122 + return d;
123 +}
124 +
125 +inline __device__ float2 fma(float a, float2 b, float2 c) {
126 + float2 d;
127 + d.x = fma(a, b.x, c.x);
128 + d.y = fma(a, b.y, c.y);
129 + return d;
130 +}
131 +
132 +inline __device__ float4 fma(float4 a, float4 b, float4 c) {
133 + float4 d;
134 + d.x = fma(a.x, b.x, c.x);
135 + d.y = fma(a.y, b.y, c.y);
136 + d.z = fma(a.z, b.z, c.z);
137 + d.w = fma(a.w, b.w, c.w);
138 + return d;
139 +}
140 +
141 +inline __device__ float4 fma(float a, float4 b, float4 c) {
142 + float4 d;
143 + d.x = fma(a, b.x, c.x);
144 + d.y = fma(a, b.y, c.y);
145 + d.z = fma(a, b.z, c.z);
146 + d.w = fma(a, b.w, c.w);
147 + return d;
148 +}
149 +
150 +inline __device__ Float4_ fma(float a, Float4_ b, Float4_ c) {
151 + Float4_ d;
152 + d.x = fma(a, b.x, c.x);
153 + d.y = fma(a, b.y, c.y);
154 + return d;
155 +}
156 +
157 +inline __device__ Float8_ fma(float a, Float8_ b, Float8_ c) {
158 + Float8_ d;
159 + d.x = fma(a, b.x, c.x);
160 + d.y = fma(a, b.y, c.y);
161 + d.z = fma(a, b.z, c.z);
162 + d.w = fma(a, b.w, c.w);
163 + return d;
164 +}
165 +
166 +// Vector sum.
167 +template<>
168 +inline __device__ float sum(float v) {
169 + return v;
170 +}
171 +
172 +template<>
173 +inline __device__ float sum(float2 v) {
174 + return v.x + v.y;
175 +}
176 +
177 +template<>
178 +inline __device__ float sum(float4 v) {
179 + return v.x + v.y + v.z + v.w;
180 +}
181 +
182 +template<>
183 +inline __device__ float sum(Float4_ v) {
184 + return v.x.x + v.x.y + v.y.x + v.y.y;
185 +}
186 +
187 +template<>
188 +inline __device__ float sum(Float8_ v) {
189 + return v.x.x + v.x.y + v.y.x + v.y.y + v.z.x + v.z.y + v.w.x + v.w.y;
190 +}
191 +
192 +// Vector dot product.
193 +inline __device__ float dot(float a, float b) {
194 + return a * b;
195 +}
196 +
197 +inline __device__ float dot(float2 a, float2 b) {
198 + float2 c = mul<float2, float2, float2>(a, b);
199 + return c.x + c.y;
200 +}
201 +
202 +inline __device__ float dot(Float4_ a, Float4_ b) {
203 + float2 acc = mul<float2, float2, float2>(a.x, b.x);
204 + acc = fma(a.y, b.y, acc);
205 + return acc.x + acc.y;
206 +}
207 +
208 +inline __device__ float dot(Float8_ a, Float8_ b) {
209 + float2 acc = mul<float2, float2, float2>(a.x, b.x);
210 + acc = fma(a.y, b.y, acc);
211 + acc = fma(a.z, b.z, acc);
212 + acc = fma(a.w, b.w, acc);
213 + return acc.x + acc.y;
214 +}
215 +
216 +// From float to float.
217 +inline __device__ void from_float(float& dst, float src) {
218 + dst = src;
219 +}
220 +
221 +inline __device__ void from_float(float2& dst, float2 src) {
222 + dst = src;
223 +}
224 +
225 +inline __device__ void from_float(float4& dst, float4 src) {
226 + dst = src;
227 +}
228 +
229 +// From float to float.
230 +inline __device__ float to_float(float u) {
231 + return u;
232 +}
233 +
234 +inline __device__ float2 to_float(float2 u) {
235 + return u;
236 +}
237 +
238 +inline __device__ float4 to_float(float4 u) {
239 + return u;
240 +}
241 +
242 +inline __device__ Float4_ to_float(Float4_ u) {
243 + return u;
244 +}
245 +
246 +inline __device__ Float8_ to_float(Float8_ u) {
247 + return u;
248 +}
249 +
250 +} // namespace cacheflow
csrc/attention_kernels.cu deleted
-896
@@ -1,896 +0,0 @@
1 -#include <torch/extension.h>
2 -#include <ATen/cuda/CUDAContext.h>
3 -
4 -#include "attention_utils.h"
5 -#include "cuda_primitives.h"
6 -#include "reduction_utils.h"
7 -
8 -#include <algorithm>
9 -
10 -#define WARP_SIZE 32
11 -#define MAX(a, b) ((a) > (b) ? (a) : (b))
12 -#define MIN(a, b) ((a) < (b) ? (a) : (b))
13 -
14 -namespace cacheflow {
15 -
16 -// Grid: (num_heads, num_seqs).
17 -template<
18 - typename scalar_t,
19 - int HEAD_SIZE,
20 - int BLOCK_SIZE,
21 - int NUM_THREADS>
22 -__global__ void single_query_cached_kv_attention_kernel(
23 - scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
24 - const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
25 - const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
26 - const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
27 - const float scale,
28 - const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
29 - const int* __restrict__ context_lens, // [num_seqs]
30 - const int max_num_blocks_per_seq,
31 - const int q_stride) {
32 - constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
33 - constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
34 - constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
35 - const int thread_idx = threadIdx.x;
36 - const int warp_idx = thread_idx / WARP_SIZE;
37 - const int lane = thread_idx % WARP_SIZE;
38 -
39 - const int head_idx = blockIdx.x;
40 - const int num_heads = gridDim.x;
41 - const int seq_idx = blockIdx.y;
42 -
43 - // A vector type to store a part of a key or a query.
44 - // The vector size is configured in such a way that the threads in a thread group
45 - // fetch or compute 16 bytes at a time.
46 - // For example, if the size of a thread group is 4 and the data type is half,
47 - // then the vector size is 16 / (4 * sizeof(half)) == 2.
48 - constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
49 - using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
50 - using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
51 -
52 - constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
53 - constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
54 -
55 - const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
56 - const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
57 -
58 - // Load the query to registers.
59 - // Each thread in a thread group has a different part of the query.
60 - // For example, if the the thread group size is 4, then the first thread in the group
61 - // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
62 - // th vectors of the query, and so on.
63 - // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
64 - const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
65 - Q_vec q_vecs[NUM_VECS_PER_THREAD];
66 -#pragma unroll
67 - for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
68 - const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
69 - q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
70 - }
71 -
72 - // Memory planning.
73 - extern __shared__ char shared_mem[];
74 - // NOTE(woosuk): We use FP32 logits and accumulation.
75 - float *logits = reinterpret_cast<float*>(shared_mem);
76 - // Workspace for reduction.
77 - __shared__ float red_smem[2 * NUM_WARPS];
78 -
79 - // x == THREAD_GROUP_SIZE * VEC_SIZE
80 - // Each thread group fetches x elements from the key at a time.
81 - constexpr int x = 16 / sizeof(scalar_t);
82 - float qk_max = -FLT_MAX;
83 -
84 - const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
85 - const int context_len = context_lens[seq_idx];
86 - const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
87 -
88 - // Iterate over the key blocks.
89 - // Each warp fetches a block of keys for each iteration.
90 - // Each thread group in a warp fetches a key from the block, and computes
91 - // dot product with the query.
92 - for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
93 - const int physical_block_number = block_table[block_idx];
94 -
95 - // Load a key to registers.
96 - // Each thread in a thread group has a different part of the key.
97 - // For example, if the the thread group size is 4, then the first thread in the group
98 - // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
99 - // vectors of the key, and so on.
100 - for (int i = 0; i < NUM_TOKENS_PER_THREAD_GROUP; i++) {
101 - const int physical_block_offset = (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;
102 - const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
103 - K_vec k_vecs[NUM_VECS_PER_THREAD];
104 -
105 -#pragma unroll
106 - for (int j = 0; j < NUM_VECS_PER_THREAD; j++) {
107 - const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
108 - + head_idx * HEAD_SIZE * BLOCK_SIZE
109 - + physical_block_offset * x;
110 - const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
111 - const int offset1 = (vec_idx * VEC_SIZE) / x;
112 - const int offset2 = (vec_idx * VEC_SIZE) % x;
113 - k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
114 - }
115 -
116 - // Compute dot product.
117 - // This includes a reduction across the threads in the same thread group.
118 - const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
119 - const bool mask = token_idx >= context_len;
120 -
121 - if (thread_group_offset == 0) {
122 - // Store the partial reductions to shared memory.
123 - // NOTE(woosuk): It is required to zero out the masked logits.
124 - logits[token_idx] = mask ? 0.f : qk;
125 - // Update the max value.
126 - qk_max = mask ? qk_max : fmaxf(qk_max, qk);
127 - }
128 - }
129 - }
130 -
131 - // Perform reduction across the threads in the same warp to get the
132 - // max qk value for each "warp" (not across the thread block yet).
133 - // The 0-th thread of each thread group already has its max qk value.
134 -#pragma unroll
135 - for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
136 - qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
137 - }
138 - if (lane == 0) {
139 - red_smem[warp_idx] = qk_max;
140 - }
141 - __syncthreads();
142 -
143 - // TODO(woosuk): Refactor this part.
144 - // Get the max qk value for the sequence.
145 - qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
146 -#pragma unroll
147 - for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
148 - qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
149 - }
150 - // Broadcast the max qk value to all threads.
151 - qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
152 -
153 - // Get the sum of the exp values.
154 - float exp_sum = 0.f;
155 - for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
156 - float val = __expf(logits[i] - qk_max);
157 - logits[i] = val;
158 - exp_sum += val;
159 - }
160 - exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
161 -
162 - // Compute softmax.
163 - const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
164 - for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
165 - logits[i] *= inv_sum;
166 - }
167 - __syncthreads();
168 -
169 - // Each thread will fetch 16 bytes from the value cache at a time.
170 - constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
171 - using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
172 - using L_vec = typename FloatVec<V_vec>::Type;
173 -
174 - constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
175 - constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
176 - constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
177 -
178 - float accs[NUM_ROWS_PER_THREAD];
179 -#pragma unroll
180 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
181 - accs[i] = 0.f;
182 - }
183 -
184 - for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
185 - const int physical_block_number = block_table[block_idx];
186 - const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
187 - const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
188 - L_vec logits_vec = *reinterpret_cast<L_vec*>(logits + token_idx);
189 -
190 - const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
191 - + head_idx * HEAD_SIZE * BLOCK_SIZE;
192 -#pragma unroll
193 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
194 - const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
195 - if (row_idx < HEAD_SIZE) {
196 - const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
197 - V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
198 - accs[i] += dot(logits_vec, cast_to_float(v_vec));
199 - }
200 - }
201 - }
202 -
203 - // Perform reduction within each warp.
204 -#pragma unroll
205 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
206 - float acc = accs[i];
207 -#pragma unroll
208 - for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
209 - acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
210 - }
211 - accs[i] = acc;
212 - }
213 -
214 - // NOTE(woosuk): A barrier is required because the shared memory space for logits
215 - // is reused for the output.
216 - __syncthreads();
217 -
218 - // Perform reduction across warps.
219 - float* out_smem = reinterpret_cast<float*>(shared_mem);
220 -#pragma unroll
221 - for (int i = NUM_WARPS; i > 1; i /= 2) {
222 - int mid = i / 2;
223 - // Upper warps write to shared memory.
224 - if (warp_idx >= mid && warp_idx < i) {
225 - float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
226 -#pragma unroll
227 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
228 - const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
229 - if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
230 - dst[row_idx] = accs[i];
231 - }
232 - }
233 - }
234 - __syncthreads();
235 -
236 - // Lower warps update the output.
237 - if (warp_idx < mid) {
238 - const float* src = &out_smem[warp_idx * HEAD_SIZE];
239 -#pragma unroll
240 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
241 - const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
242 - if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
243 - accs[i] += src[row_idx];
244 - }
245 - }
246 - }
247 - __syncthreads();
248 - }
249 -
250 - // Write the final output.
251 - if (warp_idx == 0) {
252 - scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
253 -#pragma unroll
254 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
255 - const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
256 - if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
257 - convert_from_float(*(out_ptr + row_idx), accs[i]);
258 - }
259 - }
260 - }
261 -}
262 -
263 -} // namespace cacheflow
264 -
265 -#define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
266 - cacheflow::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
267 - <<<grid, block, shared_mem_size, stream>>>( \
268 - out_ptr, \
269 - query_ptr, \
270 - key_cache_ptr, \
271 - value_cache_ptr, \
272 - scale, \
273 - block_tables_ptr, \
274 - context_lens_ptr, \
275 - max_num_blocks_per_seq, \
276 - query_stride);
277 -
278 -// TODO(woosuk): Tune NUM_THREADS.
279 -template<
280 - typename T,
281 - int BLOCK_SIZE,
282 - int NUM_THREADS = 128>
283 -void single_query_cached_kv_attention_launcher(
284 - torch::Tensor& out,
285 - torch::Tensor& query,
286 - torch::Tensor& key_cache,
287 - torch::Tensor& value_cache,
288 - float scale,
289 - torch::Tensor& block_tables,
290 - torch::Tensor& context_lens,
291 - int max_context_len) {
292 - int num_seqs = query.size(0);
293 - int num_heads = query.size(1);
294 - int head_size = query.size(2);
295 - int max_num_blocks_per_seq = block_tables.size(1);
296 - int query_stride = query.stride(0);
297 -
298 - int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
299 - assert(head_size % thread_group_size == 0);
300 -
301 - T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
302 - T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
303 - T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
304 - T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
305 - int* block_tables_ptr = block_tables.data_ptr<int>();
306 - int* context_lens_ptr = context_lens.data_ptr<int>();
307 -
308 - constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
309 - int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
310 - int logits_size = padded_max_context_len * sizeof(float);
311 - int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
312 - int shared_mem_size = std::max(logits_size, outputs_size);
313 -
314 - dim3 grid(num_heads, num_seqs);
315 - dim3 block(NUM_THREADS);
316 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
317 - switch (head_size) {
318 - case 32:
319 - LAUNCH_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
320 - break;
321 - case 64:
322 - LAUNCH_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
323 - break;
324 - case 80:
325 - LAUNCH_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
326 - break;
327 - case 96:
328 - LAUNCH_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
329 - break;
330 - case 128:
331 - LAUNCH_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
332 - break;
333 - case 160:
334 - LAUNCH_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
335 - break;
336 - case 192:
337 - LAUNCH_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
338 - break;
339 - case 256:
340 - LAUNCH_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
341 - break;
342 - default:
343 - assert(false);
344 - break;
345 - }
346 -}
347 -
348 -#define CALL_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
349 - single_query_cached_kv_attention_launcher<T, BLOCK_SIZE>( \
350 - out, \
351 - query, \
352 - key_cache, \
353 - value_cache, \
354 - scale, \
355 - block_tables, \
356 - context_lens, \
357 - max_context_len);
358 -
359 -void single_query_cached_kv_attention(
360 - torch::Tensor& out, // [num_seqs, num_heads, head_size]
361 - torch::Tensor& query, // [num_seqs, num_heads, head_size]
362 - torch::Tensor& key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
363 - torch::Tensor& value_cache, // [num_blocks, num_heads, head_size, block_size]
364 - float scale,
365 - torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
366 - torch::Tensor& context_lens, // [num_seqs]
367 - int block_size,
368 - int max_context_len) {
369 - // TODO(woosuk): Support BF16.
370 - if (query.element_size() == 2) {
371 - // Half.
372 - if (block_size == 1) {
373 - CALL_KERNEL_LAUNCHER(uint16_t, 1);
374 - } else if (block_size == 2) {
375 - CALL_KERNEL_LAUNCHER(uint16_t, 2);
376 - } else if (block_size == 4) {
377 - CALL_KERNEL_LAUNCHER(uint16_t, 4);
378 - } else if (block_size == 8) {
379 - CALL_KERNEL_LAUNCHER(uint16_t, 8);
380 - } else if (block_size == 16) {
381 - CALL_KERNEL_LAUNCHER(uint16_t, 16);
382 - } else if (block_size == 32) {
383 - CALL_KERNEL_LAUNCHER(uint16_t, 32);
384 - } else if (block_size == 64) {
385 - CALL_KERNEL_LAUNCHER(uint16_t, 64);
386 - } else if (block_size == 128) {
387 - CALL_KERNEL_LAUNCHER(uint16_t, 128);
388 - } else if (block_size == 256) {
389 - CALL_KERNEL_LAUNCHER(uint16_t, 256);
390 - } else {
391 - assert(false);
392 - }
393 - } else {
394 - // Float.
395 - assert(false);
396 - }
397 -}
398 -
399 -// namespace cacheflow {
400 -
401 -// // Grid: (num_heads, num_query_tokens).
402 -// template<
403 -// typename scalar_t,
404 -// int HEAD_SIZE,
405 -// int BLOCK_SIZE,
406 -// int NUM_THREADS>
407 -// __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
408 -// scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
409 -// const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
410 -// const int seq_start_idx,
411 -// const int seq_len,
412 -// const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
413 -// const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
414 -// const float scale,
415 -// const int* __restrict__ block_table, // [num_seqs, max_num_blocks_per_seq]
416 -// const int context_len,
417 -// const int max_num_blocks_per_seq,
418 -// const int q_stride) {
419 -// constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
420 -// constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
421 -// const int thread_idx = threadIdx.x;
422 -// const int warp_idx = thread_idx / WARP_SIZE;
423 -// const int lane = thread_idx % WARP_SIZE;
424 -
425 -// const int head_idx = blockIdx.x;
426 -// const int num_heads = gridDim.x;
427 -// const int seq_idx = blockIdx.y;
428 -
429 -// // A vector type to store a part of a key or a query.
430 -// // The vector size is configured in such a way that the threads in a thread group
431 -// // fetch or comput 16 bytes at a time.
432 -// // For example, if the size of a thread group is 4 and the data type is half,
433 -// // then the vector size is 16 / (4 * sizeof(half)) == 2.
434 -// constexpr int VEC_SIZE = 16 / (THREAD_GROUP_SIZE * sizeof(scalar_t));
435 -// using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
436 -// using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
437 -
438 -// constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
439 -// constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
440 -
441 -// const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
442 -// const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
443 -
444 -// // Load the query to registers.
445 -// // Each thread in a thread group has a different part of the query.
446 -// // For example, if the the thread group size is 4, then the first thread in the group
447 -// // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
448 -// // th vectors of the query, and so on.
449 -// // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
450 -// const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
451 -// Q_vec q_vecs[NUM_VECS_PER_THREAD];
452 -// #pragma unroll
453 -// for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
454 -// const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
455 -// q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
456 -// }
457 -
458 -// // Memory planning.
459 -// extern __shared__ char shared_mem[];
460 -// // NOTE(woosuk): We use FP32 logits and accumulation.
461 -// float *logits = reinterpret_cast<float*>(shared_mem);
462 -// // Workspace for reduction.
463 -// __shared__ float red_smem[2 * NUM_WARPS];
464 -
465 -// // x == THREAD_GROUP_SIZE * VEC_SIZE
466 -// // Each thread group fetches x elements from the key at a time.
467 -// constexpr int x = 16 / sizeof(scalar_t);
468 -// float qk_max = -FLT_MAX;
469 -
470 -// const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
471 -// const int mask_boundary = context_len - seq_len + 1 + (seq_idx - seq_start_idx);
472 -
473 -// // Iterate over the key blocks.
474 -// // Each warp fetches a block of keys for each iteration.
475 -// // Each thread group in a warp fetches a key from the block, and computes
476 -// // dot product with the query.
477 -// for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
478 -// const int physical_block_number = block_table[block_idx];
479 -// const int physical_block_offset = thread_group_idx % BLOCK_SIZE;
480 -// const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
481 -
482 -// // Load a key to registers.
483 -// // Each thread in a thread group has a different part of the key.
484 -// // For example, if the the thread group size is 4, then the first thread in the group
485 -// // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
486 -// // vectors of the key, and so on.
487 -// K_vec k_vecs[NUM_VECS_PER_THREAD];
488 -// #pragma unroll
489 -// for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
490 -// const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
491 -// + head_idx * HEAD_SIZE * BLOCK_SIZE
492 -// + physical_block_offset * x;
493 -// const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
494 -// const int offset1 = (vec_idx * VEC_SIZE) / x;
495 -// const int offset2 = (vec_idx * VEC_SIZE) % x;
496 -// k_vecs[i] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
497 -// }
498 -
499 -// // Compute dot product.
500 -// // This includes a reduction across the threads in the same thread group.
501 -// const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
502 -// const bool mask = token_idx >= mask_boundary;
503 -
504 -// if (thread_group_offset == 0) {
505 -// // Store the partial reductions to shared memory.
506 -// // NOTE(woosuk): It is required to zero out the masked logits.
507 -// logits[token_idx] = mask ? 0.f : qk;
508 -// // Update the max value.
509 -// qk_max = mask ? qk_max : fmaxf(qk_max, qk);
510 -// }
511 -// }
512 -
513 -// // Perform reduction across the threads in the same warp to get the
514 -// // max qk value for each "warp" (not across the thread block yet).
515 -// // The 0-th thread of each thread group already has its max qk value.
516 -// #pragma unroll
517 -// for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
518 -// qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
519 -// }
520 -// if (lane == 0) {
521 -// red_smem[warp_idx] = qk_max;
522 -// }
523 -// __syncthreads();
524 -
525 -// // TODO(woosuk): Refactor this part.
526 -// // Get the max qk value for the sequence.
527 -// qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
528 -// #pragma unroll
529 -// for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
530 -// qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
531 -// }
532 -// // Broadcast the max qk value to all threads.
533 -// qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
534 -
535 -// // Get the sum of the exp values.
536 -// float exp_sum = 0.f;
537 -// for (int i = thread_idx; i < mask_boundary; i += NUM_THREADS) {
538 -// float val = __expf(logits[i] - qk_max);
539 -// logits[i] = val;
540 -// exp_sum += val;
541 -// }
542 -// exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
543 -
544 -// // Compute softmax.
545 -// const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
546 -// for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
547 -// logits[i] *= inv_sum;
548 -// }
549 -// __syncthreads();
550 -
551 -// // Each thread will fetch 16 bytes from the value cache at a time.
552 -// constexpr int V_VEC_SIZE = 16 / sizeof(scalar_t);
553 -// using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
554 -// using L_vec = typename FloatVec<V_vec>::Type;
555 -
556 -// constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
557 -// constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
558 -// constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
559 -
560 -// float accs[NUM_ROWS_PER_THREAD];
561 -// #pragma unroll
562 -// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
563 -// accs[i] = 0.f;
564 -// }
565 -
566 -// for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
567 -// const int physical_block_number = block_table[block_idx];
568 -// const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
569 -// const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
570 -// L_vec logits_vec = *reinterpret_cast<L_vec*>(logits + token_idx);
571 -
572 -// const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
573 -// + head_idx * HEAD_SIZE * BLOCK_SIZE;
574 -// #pragma unroll
575 -// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
576 -// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
577 -// if (row_idx < HEAD_SIZE) {
578 -// const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
579 -// V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
580 -// accs[i] += dot(logits_vec, cast_to_float(v_vec));
581 -// }
582 -// }
583 -// }
584 -
585 -// // Perform reduction within each warp.
586 -// #pragma unroll
587 -// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
588 -// float acc = accs[i];
589 -// #pragma unroll
590 -// for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
591 -// acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
592 -// }
593 -// accs[i] = acc;
594 -// }
595 -
596 -// // NOTE(woosuk): A barrier is required because the shared memory space for logits
597 -// // is reused for the output.
598 -// __syncthreads();
599 -
600 -// // Perform reduction across warps.
601 -// float* out_smem = reinterpret_cast<float*>(shared_mem);
602 -// #pragma unroll
603 -// for (int i = NUM_WARPS; i > 1; i /= 2) {
604 -// int mid = i / 2;
605 -// // Upper warps write to shared memory.
606 -// if (warp_idx >= mid && warp_idx < i) {
607 -// float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
608 -// #pragma unroll
609 -// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
610 -// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
611 -// if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
612 -// dst[row_idx] = accs[i];
613 -// }
614 -// }
615 -// }
616 -// __syncthreads();
617 -
618 -// // Lower warps update the output.
619 -// if (warp_idx < mid) {
620 -// const float* src = &out_smem[warp_idx * HEAD_SIZE];
621 -// #pragma unroll
622 -// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
623 -// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
624 -// if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
625 -// accs[i] += src[row_idx];
626 -// }
627 -// }
628 -// }
629 -// __syncthreads();
630 -// }
631 -
632 -// // Write the final output.
633 -// if (warp_idx == 0) {
634 -// scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
635 -// #pragma unroll
636 -// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
637 -// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
638 -// if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
639 -// convert_from_float(*(out_ptr + row_idx), accs[i]);
640 -// }
641 -// }
642 -// }
643 -// }
644 -
645 -
646 -// // Grid: (num_heads, num_query_tokens).
647 -// template<
648 -// typename scalar_t,
649 -// int HEAD_SIZE,
650 -// int BLOCK_SIZE,
651 -// int NUM_THREADS>
652 -// __global__ void multi_query_cached_kv_attention_kernel(
653 -// const int* cu_query_lens, // [num_prompts+1]
654 -// const int* seq_prompt_mapping, // [num_seqs] mapping from seq_idx to prompt_idx
655 -// scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
656 -// const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
657 -// const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
658 -// const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
659 -// const float scale,
660 -// const int* __restrict__ block_tables, // [num_prompts, max_num_blocks_per_seq]
661 -// const int* __restrict__ context_lens, // [num_prompts]
662 -// const int max_num_blocks_per_seq,
663 -// const int q_stride) {
664 -// const int seq_idx = blockIdx.y;
665 -// const int prompt_idx = seq_prompt_mapping[seq_idx];
666 -// const int seq_start_idx = cu_query_lens[prompt_idx];
667 -// const int seq_len = cu_query_lens[prompt_idx + 1] - seq_start_idx;
668 -// const int* block_table = block_tables + prompt_idx * max_num_blocks_per_seq;
669 -// const int context_len = context_lens[prompt_idx];
670 -// multi_query_cached_kv_attention_kernel_unoptimized_<
671 -// scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
672 -// out,
673 -// q,
674 -// seq_start_idx,
675 -// seq_len,
676 -// k_cache,
677 -// v_cache,
678 -// scale,
679 -// block_table,
680 -// context_len,
681 -// max_num_blocks_per_seq,
682 -// q_stride);
683 -// }
684 -
685 -// } // namespace cacheflow
686 -
687 -// #define LAUNCH_MULTI_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
688 -// cacheflow::multi_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
689 -// <<<grid, block, shared_mem_size, stream>>>( \
690 -// cu_query_lens_ptr, \
691 -// seq_prompt_mapping_ptr, \
692 -// out_ptr, \
693 -// query_ptr, \
694 -// key_cache_ptr, \
695 -// value_cache_ptr, \
696 -// scale, \
697 -// block_tables_ptr, \
698 -// context_lens_ptr, \
699 -// max_num_blocks_per_seq, \
700 -// query_stride);
701 -
702 -
703 -// // TODO(woosuk): Tune NUM_THREADS.
704 -// template<
705 -// typename T,
706 -// int BLOCK_SIZE,
707 -// int NUM_THREADS = 128>
708 -// void multi_query_cached_kv_attention_launcher(
709 -// torch::Tensor& cu_query_lens,
710 -// torch::Tensor& seq_prompt_mapping,
711 -// torch::Tensor& out,
712 -// torch::Tensor& query,
713 -// torch::Tensor& key_cache,
714 -// torch::Tensor& value_cache,
715 -// float scale,
716 -// torch::Tensor& block_tables,
717 -// torch::Tensor& context_lens,
718 -// int max_context_len) {
719 -// int num_seqs = query.size(0);
720 -// int num_heads = query.size(1);
721 -// int head_size = query.size(2);
722 -// int max_num_blocks_per_seq = block_tables.size(1);
723 -// int query_stride = query.stride(0);
724 -
725 -// int* cu_query_lens_ptr = cu_query_lens.data_ptr<int>();
726 -// int* seq_prompt_mapping_ptr = seq_prompt_mapping.data_ptr<int>();
727 -// T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
728 -// T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
729 -// T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
730 -// T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
731 -// int* block_tables_ptr = block_tables.data_ptr<int>();
732 -// int* context_lens_ptr = context_lens.data_ptr<int>();
733 -
734 -// constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
735 -// int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
736 -// int logits_size = padded_max_context_len * sizeof(float);
737 -// int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
738 -// int shared_mem_size = std::max(logits_size, outputs_size);
739 -
740 -// dim3 grid(num_heads, num_seqs);
741 -// dim3 block(NUM_THREADS);
742 -// const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
743 -// switch (head_size) {
744 -// case 32:
745 -// LAUNCH_MULTI_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
746 -// break;
747 -// case 64:
748 -// LAUNCH_MULTI_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
749 -// break;
750 -// case 80:
751 -// LAUNCH_MULTI_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
752 -// break;
753 -// case 96:
754 -// LAUNCH_MULTI_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
755 -// break;
756 -// case 128:
757 -// LAUNCH_MULTI_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
758 -// break;
759 -// case 160:
760 -// LAUNCH_MULTI_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
761 -// break;
762 -// case 192:
763 -// LAUNCH_MULTI_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
764 -// break;
765 -// case 256:
766 -// LAUNCH_MULTI_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
767 -// break;
768 -// default:
769 -// assert(false);
770 -// break;
771 -// }
772 -// }
773 -
774 -// void multi_query_cached_kv_attention(
775 -// torch::Tensor& cu_query_lens,
776 -// torch::Tensor& out,
777 -// torch::Tensor& query,
778 -// torch::Tensor& key_cache,
779 -// torch::Tensor& value_cache,
780 -// float scale,
781 -// torch::Tensor& block_tables,
782 -// torch::Tensor& context_lens,
783 -// int block_size,
784 -// int max_context_len) {
785 -
786 -// torch::Tensor query_lens = cu_query_lens.to(torch::kCPU);
787 -
788 -// int num_queries = query_lens.size(0) - 1;
789 -// const int* query_lens_ptr = query_lens.data_ptr<int>();
790 -// int num_seqs = query.size(0);
791 -
792 -// torch::Tensor cpu_tensor = torch::empty({num_seqs}, torch::dtype(torch::kInt32));
793 -// auto accessor = cpu_tensor.accessor<int32_t, 1>();
794 -// for (int i = 0, query_cursor = 0; i < num_seqs; ++i) {
795 -// if (i >= query_lens_ptr[query_cursor + 1]) {
796 -// ++query_cursor;
797 -// }
798 -// accessor[i] = query_cursor;
799 -// }
800 -
801 -// // TODO(suquark): This can be slow, as it to(torch::kCPU) and to(torch::kCUDA)
802 -// // implicitly synchronizes the CPU and GPU. And we can avoid this issue by giving
803 -// // the mapping as an input parameter. Let's do this optimization in a later PR.
804 -// torch::Tensor seq_prompt_mapping = cpu_tensor.to(torch::kCUDA);
805 -
806 -// // TODO(woosuk): Support BF16.
807 -// if (query.element_size() == 2) {
808 -// // Half.
809 -// if (block_size == 8) {
810 -// multi_query_cached_kv_attention_launcher<uint16_t, 8>(
811 -// cu_query_lens,
812 -// seq_prompt_mapping,
813 -// out,
814 -// query,
815 -// key_cache,
816 -// value_cache,
817 -// scale,
818 -// block_tables,
819 -// context_lens,
820 -// max_context_len);
821 -// } else if (block_size == 16) {
822 -// multi_query_cached_kv_attention_launcher<uint16_t, 16>(
823 -// cu_query_lens,
824 -// seq_prompt_mapping,
825 -// out,
826 -// query,
827 -// key_cache,
828 -// value_cache,
829 -// scale,
830 -// block_tables,
831 -// context_lens,
832 -// max_context_len);
833 -// } else if (block_size == 32) {
834 -// multi_query_cached_kv_attention_launcher<uint16_t, 32>(
835 -// cu_query_lens,
836 -// seq_prompt_mapping,
837 -// out,
838 -// query,
839 -// key_cache,
840 -// value_cache,
841 -// scale,
842 -// block_tables,
843 -// context_lens,
844 -// max_context_len);
845 -// } else {
846 -// assert(false);
847 -// }
848 -// } else if (query.element_size() == 4) {
849 -// // Float.
850 -// if (block_size == 8) {
851 -// multi_query_cached_kv_attention_launcher<float, 8>(
852 -// cu_query_lens,
853 -// seq_prompt_mapping,
854 -// out,
855 -// query,
856 -// key_cache,
857 -// value_cache,
858 -// scale,
859 -// block_tables,
860 -// context_lens,
861 -// max_context_len);
862 -// } else if (block_size == 16) {
863 -// multi_query_cached_kv_attention_launcher<float, 16>(
864 -// cu_query_lens,
865 -// seq_prompt_mapping,
866 -// out,
867 -// query,
868 -// key_cache,
869 -// value_cache,
870 -// scale,
871 -// block_tables,
872 -// context_lens,
873 -// max_context_len);
874 -// } else if (block_size == 32) {
875 -// multi_query_cached_kv_attention_launcher<float, 32>(
876 -// cu_query_lens,
877 -// seq_prompt_mapping,
878 -// out,
879 -// query,
880 -// key_cache,
881 -// value_cache,
882 -// scale,
883 -// block_tables,
884 -// context_lens,
885 -// max_context_len);
886 -// } else {
887 -// assert(false);
888 -// }
889 -// } else {
890 -// assert(false);
891 -// }
892 -// }
893 -
894 -#undef WARP_SIZE
895 -#undef MAX
896 -#undef MIN
csrc/attention_utils.h deleted
-165
@@ -1,165 +0,0 @@
1 -#pragma once
2 -
3 -#include "cuda_primitives.h"
4 -
5 -#include <float.h>
6 -#include <type_traits>
7 -
8 -#define MMHA_USE_FP32_ACUM_FOR_FMA
9 -#define MMHA_USE_FP32_ACUM_FOR_OUT
10 -
11 -namespace cacheflow {
12 -
13 -// A vector type to store Q, K, V elements.
14 -template<typename T, int VEC_SIZE>
15 -struct Vec {};
16 -template<>
17 -struct Vec<float, 1> {
18 - using Type = float;
19 -};
20 -template<>
21 -struct Vec<float, 2> {
22 - using Type = float2;
23 -};
24 -template<>
25 -struct Vec<float, 4> {
26 - using Type = float4;
27 -};
28 -template<>
29 -struct Vec<uint16_t, 1> {
30 - using Type = uint16_t;
31 -};
32 -template<>
33 -struct Vec<uint16_t, 2> {
34 - using Type = uint32_t;
35 -};
36 -template<>
37 -struct Vec<uint16_t, 4> {
38 - using Type = uint2;
39 -};
40 -template<>
41 -struct Vec<uint16_t, 8> {
42 - using Type = uint4;
43 -};
44 -
45 -template<typename T>
46 -struct FloatVec {};
47 -template<>
48 -struct FloatVec<float> {
49 - using Type = float;
50 -};
51 -template<>
52 -struct FloatVec<float2> {
53 - using Type = float2;
54 -};
55 -template<>
56 -struct FloatVec<float4> {
57 - using Type = float4;
58 -};
59 -template<>
60 -struct FloatVec<uint16_t> {
61 - using Type = float;
62 -};
63 -template<>
64 -struct FloatVec<uint32_t> {
65 - using Type = float2;
66 -};
67 -template<>
68 -struct FloatVec<uint2> {
69 - using Type = Float4_;
70 -};
71 -template<>
72 -struct FloatVec<uint4> {
73 - using Type = Float8_;
74 -};
75 -
76 -template<int THREADS_PER_KEY, typename K_vec, int N>
77 -inline __device__ float qk_dot_(const K_vec (&q)[N], const K_vec (&k)[N])
78 -{
79 - using K_vec_acum = typename FloatVec<K_vec>::Type;
80 - // Compute the parallel products for Q*K^T (treat vector lanes separately).
81 - K_vec_acum qk_vec = mul<K_vec_acum, K_vec, K_vec>(q[0], k[0]);
82 -#pragma unroll
83 - for (int ii = 1; ii < N; ++ii) {
84 - qk_vec = fma(q[ii], k[ii], qk_vec);
85 - }
86 -
87 - // Finalize the reduction across lanes.
88 - float qk = sum(qk_vec);
89 -#pragma unroll
90 - for (int mask = THREADS_PER_KEY / 2; mask >= 1; mask /= 2) {
91 - qk += __shfl_xor_sync(uint32_t(-1), qk, mask);
92 - }
93 - return qk;
94 -}
95 -
96 -////////////////////////////////////////////////////////////////////////////////////////////////////
97 -
98 -template<typename T, int THREADS_PER_KEY>
99 -struct Qk_dot {
100 - template<typename K_vec, int N>
101 - static inline __device__ float dot(const K_vec (&q)[N], const K_vec (&k)[N])
102 - {
103 - return qk_dot_<THREADS_PER_KEY>(q, k);
104 - }
105 -};
106 -
107 -////////////////////////////////////////////////////////////////////////////////////////////////////
108 -
109 -inline __device__ float4 hmma_fp32(const uint2& a, uint32_t b)
110 -{
111 - float4 c;
112 - float zero = 0.f;
113 - asm volatile("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 \n"
114 - " {%0, %1, %2, %3}, \n"
115 - " {%4, %5}, \n"
116 - " {%6}, \n"
117 - " {%7, %7, %7, %7}; \n"
118 -
119 - : "=f"(c.x), "=f"(c.y), "=f"(c.z), "=f"(c.w)
120 - : "r"(a.x) "r"(a.y), "r"(b), "f"(zero));
121 - return c;
122 -}
123 -
124 -////////////////////////////////////////////////////////////////////////////////////////////////////
125 -
126 -template<int N>
127 -inline __device__ float qk_hmma_dot_(const uint32_t (&q)[N], const uint32_t (&k)[N])
128 -{
129 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 750
130 - using K_vec_acum = typename FloatVec<uint32_t>::Type;
131 - K_vec_acum qk_vec = mul<K_vec_acum, uint32_t, uint32_t>(q[0], k[0]);
132 -#pragma unroll
133 - for (int ii = 1; ii < N; ++ii) {
134 - qk_vec = fma(q[ii], k[ii], qk_vec);
135 - }
136 -#ifdef MMHA_USE_FP32_ACUM_FOR_FMA
137 - uint32_t qk_vec_ = float2_to_half2(qk_vec);
138 - return hmma_fp32(make_uint2(qk_vec_, 0u), 0x3c003c00u).x;
139 -#else
140 - return hmma_fp32(make_uint2(qk_vec, 0u), 0x3c003c00u).x;
141 -#endif
142 -#else
143 - return 0.f;
144 -#endif
145 -}
146 -
147 -////////////////////////////////////////////////////////////////////////////////////////////////////
148 -
149 -template<>
150 -struct Qk_dot<uint16_t, 4> {
151 - template<int N>
152 - static inline __device__ float dot(const uint32_t (&q)[N], const uint32_t (&k)[N])
153 - {
154 -#if __CUDA_ARCH__ >= 750 && defined(MMHA_USE_HMMA_FOR_REDUCTION)
155 - return qk_hmma_dot_(q, k);
156 -#else
157 - return qk_dot_<4>(q, k);
158 -#endif // defined MMHA_USE_HMMA_FOR_REDUCTION
159 - }
160 -};
161 -
162 -} // namespace cacheflow
163 -
164 -#undef MMHA_USE_FP32_ACUM_FOR_FMA
165 -#undef MMHA_USE_FP32_ACUM_FOR_OUT
csrc/cuda_primitives.h deleted
-1340
@@ -1,1340 +0,0 @@
1 -#pragma once
2 -
3 -#include <stdint.h>
4 -
5 -namespace cacheflow {
6 -////////////////////////////////////////////////////////////////////////////////////////////////////
7 -
8 -struct Float8_ {
9 - float2 x;
10 - float2 y;
11 - float2 z;
12 - float2 w;
13 -};
14 -
15 -////////////////////////////////////////////////////////////////////////////////////////////////////
16 -
17 -struct Float4_ {
18 - float2 x;
19 - float2 y;
20 -};
21 -
22 -////////////////////////////////////////////////////////////////////////////////////////////////////
23 -
24 -#ifdef ENABLE_BF16
25 -struct bf16_4_t {
26 - __nv_bfloat162 x;
27 - __nv_bfloat162 y;
28 -};
29 -
30 -////////////////////////////////////////////////////////////////////////////////////////////////////
31 -
32 -struct bf16_8_t {
33 - __nv_bfloat162 x;
34 - __nv_bfloat162 y;
35 - __nv_bfloat162 z;
36 - __nv_bfloat162 w;
37 -};
38 -#endif
39 -
40 -////////////////////////////////////////////////////////////////////////////////////////////////////
41 -
42 -inline __device__ float add(float a, float b)
43 -{
44 - return a + b;
45 -}
46 -
47 -////////////////////////////////////////////////////////////////////////////////////////////////////
48 -
49 -inline __device__ float2 add(float2 a, float2 b)
50 -{
51 - float2 c;
52 - c.x = add(a.x, b.x);
53 - c.y = add(a.y, b.y);
54 - return c;
55 -}
56 -
57 -////////////////////////////////////////////////////////////////////////////////////////////////////
58 -
59 -inline __device__ float4 add(float4 a, float4 b)
60 -{
61 - float4 c;
62 - c.x = add(a.x, b.x);
63 - c.y = add(a.y, b.y);
64 - c.z = add(a.z, b.z);
65 - c.w = add(a.w, b.w);
66 - return c;
67 -}
68 -
69 -////////////////////////////////////////////////////////////////////////////////////////////////////
70 -
71 -#ifdef ENABLE_BF16
72 -inline __device__ __nv_bfloat16 add(__nv_bfloat16 a, __nv_bfloat16 b)
73 -{
74 - return a + b;
75 -}
76 -
77 -////////////////////////////////////////////////////////////////////////////////////////////////////
78 -
79 -inline __device__ __nv_bfloat162 add(__nv_bfloat162 a, __nv_bfloat162 b)
80 -{
81 - return bf16hadd2(a, b);
82 -}
83 -
84 -////////////////////////////////////////////////////////////////////////////////////////////////////
85 -
86 -inline __device__ bf16_4_t add(bf16_4_t a, bf16_4_t b)
87 -{
88 - bf16_4_t c;
89 - c.x = add(a.x, b.x);
90 - c.y = add(a.y, b.y);
91 - return c;
92 -}
93 -
94 -////////////////////////////////////////////////////////////////////////////////////////////////////
95 -
96 -inline __device__ bf16_8_t add(bf16_8_t a, bf16_8_t b)
97 -{
98 - bf16_8_t c;
99 - c.x = add(a.x, b.x);
100 - c.y = add(a.y, b.y);
101 - c.z = add(a.z, b.z);
102 - c.w = add(a.w, b.w);
103 - return c;
104 -}
105 -#endif // ENABLE_BF16
106 -
107 -////////////////////////////////////////////////////////////////////////////////////////////////////
108 -
109 -inline __device__ uint16_t add(uint16_t a, uint16_t b)
110 -{
111 - uint16_t c;
112 - asm volatile("add.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
113 - return c;
114 -}
115 -
116 -////////////////////////////////////////////////////////////////////////////////////////////////////
117 -
118 -inline __device__ uint32_t add(uint32_t a, uint32_t b)
119 -{
120 - uint32_t c;
121 - asm volatile("add.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
122 - return c;
123 -}
124 -
125 -////////////////////////////////////////////////////////////////////////////////////////////////////
126 -
127 -inline __device__ uint2 add(uint2 a, uint2 b)
128 -{
129 - uint2 c;
130 - c.x = add(a.x, b.x);
131 - c.y = add(a.y, b.y);
132 - return c;
133 -}
134 -
135 -////////////////////////////////////////////////////////////////////////////////////////////////////
136 -
137 -inline __device__ uint4 add(uint4 a, uint4 b)
138 -{
139 - uint4 c;
140 - c.x = add(a.x, b.x);
141 - c.y = add(a.y, b.y);
142 - c.z = add(a.z, b.z);
143 - c.w = add(a.w, b.w);
144 - return c;
145 -}
146 -
147 -////////////////////////////////////////////////////////////////////////////////////////////////////
148 -
149 -inline __device__ uint16_t float_to_half(float f)
150 -{
151 - union {
152 - uint32_t u32;
153 - uint16_t u16[2];
154 - } tmp;
155 -#if 0 && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 // Is it better?
156 - float zero = 0.f;
157 - asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(zero), "f"(f));
158 -#else
159 - asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f));
160 -#endif
161 - return tmp.u16[0];
162 -}
163 -
164 -////////////////////////////////////////////////////////////////////////////////////////////////////
165 -
166 -inline __device__ uint32_t float2_to_half2(float2 f)
167 -{
168 - union {
169 - uint32_t u32;
170 - uint16_t u16[2];
171 - } tmp;
172 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
173 - asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(f.y), "f"(f.x));
174 -#else
175 - asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
176 - asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
177 -#endif
178 - return tmp.u32;
179 -}
180 -
181 -////////////////////////////////////////////////////////////////////////////////////////////////////
182 -
183 -inline __device__ float half_to_float(uint16_t h)
184 -{
185 - float f;
186 - asm volatile("cvt.f32.f16 %0, %1;\n" : "=f"(f) : "h"(h));
187 - return f;
188 -}
189 -
190 -////////////////////////////////////////////////////////////////////////////////////////////////////
191 -
192 -inline __device__ float2 half2_to_float2(uint32_t v)
193 -{
194 - uint16_t lo, hi;
195 - asm volatile("mov.b32 {%0, %1}, %2;\n" : "=h"(lo), "=h"(hi) : "r"(v));
196 - return make_float2(half_to_float(lo), half_to_float(hi));
197 -}
198 -
199 -////////////////////////////////////////////////////////////////////////////////////////////////////
200 -
201 -inline __device__ float2 add(uint32_t a, float2 fb)
202 -{
203 - float2 fa = half2_to_float2(a);
204 - return add(fa, fb);
205 -}
206 -
207 -////////////////////////////////////////////////////////////////////////////////////////////////////
208 -
209 -inline __device__ Float4_ add(uint2 a, Float4_ fb)
210 -{
211 - Float4_ fc;
212 - fc.x = add(a.x, fb.x);
213 - fc.y = add(a.y, fb.y);
214 - return fc;
215 -}
216 -
217 -////////////////////////////////////////////////////////////////////////////////////////////////////
218 -
219 -inline __device__ Float8_ add(uint4 a, Float8_ fb)
220 -{
221 - Float8_ fc;
222 - fc.x = add(a.x, fb.x);
223 - fc.y = add(a.y, fb.y);
224 - fc.z = add(a.z, fb.z);
225 - fc.w = add(a.w, fb.w);
226 - return fc;
227 -}
228 -
229 -////////////////////////////////////////////////////////////////////////////////////////////////////
230 -
231 -inline __device__ uint32_t h0_h0(uint16_t a)
232 -{
233 - uint32_t b;
234 - asm volatile("mov.b32 %0, {%1, %1};" : "=r"(b) : "h"(a));
235 - return b;
236 -}
237 -
238 -////////////////////////////////////////////////////////////////////////////////////////////////////
239 -
240 -inline __device__ float fma(float a, float b, float c)
241 -{
242 - return a * b + c;
243 -}
244 -
245 -////////////////////////////////////////////////////////////////////////////////////////////////////
246 -
247 -inline __device__ float2 fma(float2 a, float2 b, float2 c)
248 -{
249 - float2 d;
250 - d.x = fma(a.x, b.x, c.x);
251 - d.y = fma(a.y, b.y, c.y);
252 - return d;
253 -}
254 -
255 -////////////////////////////////////////////////////////////////////////////////////////////////////
256 -
257 -inline __device__ float2 fma(float a, float2 b, float2 c)
258 -{
259 - float2 d;
260 - d.x = fma(a, b.x, c.x);
261 - d.y = fma(a, b.y, c.y);
262 - return d;
263 -}
264 -
265 -////////////////////////////////////////////////////////////////////////////////////////////////////
266 -
267 -inline __device__ float4 fma(float4 a, float4 b, float4 c)
268 -{
269 - float4 d;
270 - d.x = fma(a.x, b.x, c.x);
271 - d.y = fma(a.y, b.y, c.y);
272 - d.z = fma(a.z, b.z, c.z);
273 - d.w = fma(a.w, b.w, c.w);
274 - return d;
275 -}
276 -
277 -////////////////////////////////////////////////////////////////////////////////////////////////////
278 -
279 -inline __device__ float4 fma(float a, float4 b, float4 c)
280 -{
281 - float4 d;
282 - d.x = fma(a, b.x, c.x);
283 - d.y = fma(a, b.y, c.y);
284 - d.z = fma(a, b.z, c.z);
285 - d.w = fma(a, b.w, c.w);
286 - return d;
287 -}
288 -
289 -////////////////////////////////////////////////////////////////////////////////////////////////////
290 -
291 -inline __device__ Float4_ fma(float a, Float4_ b, Float4_ c)
292 -{
293 - Float4_ d;
294 - d.x = fma(a, b.x, c.x);
295 - d.y = fma(a, b.y, c.y);
296 - return d;
297 -}
298 -
299 -////////////////////////////////////////////////////////////////////////////////////////////////////
300 -
301 -inline __device__ Float8_ fma(float a, Float8_ b, Float8_ c)
302 -{
303 - Float8_ d;
304 - d.x = fma(a, b.x, c.x);
305 - d.y = fma(a, b.y, c.y);
306 - d.z = fma(a, b.z, c.z);
307 - d.w = fma(a, b.w, c.w);
308 - return d;
309 -}
310 -
311 -////////////////////////////////////////////////////////////////////////////////////////////////////
312 -
313 -#ifdef ENABLE_BF16
314 -inline __device__ float2 add(__nv_bfloat162 a, float2 fb)
315 -{
316 - float2 fa = bf1622float2(a);
317 - return add(fa, fb);
318 -}
319 -
320 -////////////////////////////////////////////////////////////////////////////////////////////////////
321 -
322 -inline __device__ Float4_ add(bf16_4_t a, Float4_ fb)
323 -{
324 - Float4_ fc;
325 - fc.x = add(a.x, fb.x);
326 - fc.y = add(a.y, fb.y);
327 - return fc;
328 -}
329 -
330 -////////////////////////////////////////////////////////////////////////////////////////////////////
331 -
332 -inline __device__ Float8_ add(bf16_8_t a, Float8_ fb)
333 -{
334 - Float8_ fc;
335 - fc.x = add(a.x, fb.x);
336 - fc.y = add(a.y, fb.y);
337 - fc.z = add(a.z, fb.z);
338 - fc.w = add(a.w, fb.w);
339 - return fc;
340 -}
341 -#endif // ENABLE_BF16
342 -
343 -////////////////////////////////////////////////////////////////////////////////////////////////////
344 -
345 -inline __device__ uint32_t fma(uint32_t a, uint32_t b, uint32_t c)
346 -{
347 - uint32_t d;
348 - asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(d) : "r"(a), "r"(b), "r"(c));
349 - return d;
350 -}
351 -
352 -////////////////////////////////////////////////////////////////////////////////////////////////////
353 -
354 -inline __device__ uint32_t fma(uint16_t a, uint32_t b, uint32_t c)
355 -{
356 - return fma(h0_h0(a), b, c);
357 -}
358 -
359 -////////////////////////////////////////////////////////////////////////////////////////////////////
360 -
361 -inline __device__ uint2 fma(uint2 a, uint2 b, uint2 c)
362 -{
363 - uint2 d;
364 - d.x = fma(a.x, b.x, c.x);
365 - d.y = fma(a.y, b.y, c.y);
366 - return d;
367 -}
368 -
369 -////////////////////////////////////////////////////////////////////////////////////////////////////
370 -
371 -inline __device__ uint2 fma(uint16_t a, uint2 b, uint2 c)
372 -{
373 - uint32_t s = h0_h0(a);
374 - uint2 d;
375 - d.x = fma(s, b.x, c.x);
376 - d.y = fma(s, b.y, c.y);
377 - return d;
378 -}
379 -
380 -////////////////////////////////////////////////////////////////////////////////////////////////////
381 -
382 -inline __device__ uint4 fma(uint4 a, uint4 b, uint4 c)
383 -{
384 - uint4 d;
385 - d.x = fma(a.x, b.x, c.x);
386 - d.y = fma(a.y, b.y, c.y);
387 - d.z = fma(a.z, b.z, c.z);
388 - d.w = fma(a.w, b.w, c.w);
389 - return d;
390 -}
391 -
392 -////////////////////////////////////////////////////////////////////////////////////////////////////
393 -
394 -inline __device__ uint4 fma(uint16_t a, uint4 b, uint4 c)
395 -{
396 - uint32_t s = h0_h0(a);
397 - uint4 d;
398 - d.x = fma(s, b.x, c.x);
399 - d.y = fma(s, b.y, c.y);
400 - d.z = fma(s, b.z, c.z);
401 - d.w = fma(s, b.w, c.w);
402 - return d;
403 -}
404 -
405 -////////////////////////////////////////////////////////////////////////////////////////////////////
406 -
407 -inline __device__ float fma(uint16_t a, uint16_t b, float fc)
408 -{
409 - float fa = half_to_float(a);
410 - float fb = half_to_float(b);
411 - return fa * fb + fc;
412 -}
413 -
414 -////////////////////////////////////////////////////////////////////////////////////////////////////
415 -
416 -inline __device__ float2 fma(uint32_t a, uint32_t b, float2 fc)
417 -{
418 - float2 fa = half2_to_float2(a);
419 - float2 fb = half2_to_float2(b);
420 - return fma(fa, fb, fc);
421 -}
422 -
423 -////////////////////////////////////////////////////////////////////////////////////////////////////
424 -
425 -inline __device__ float2 fma(uint16_t a, uint32_t b, float2 fc)
426 -{
427 - return fma(h0_h0(a), b, fc);
428 -}
429 -
430 -////////////////////////////////////////////////////////////////////////////////////////////////////
431 -
432 -inline __device__ Float4_ fma(uint2 a, uint2 b, Float4_ fc)
433 -{
434 - Float4_ fd;
435 - fd.x = fma(a.x, b.x, fc.x);
436 - fd.y = fma(a.y, b.y, fc.y);
437 - return fd;
438 -}
439 -
440 -////////////////////////////////////////////////////////////////////////////////////////////////////
441 -
442 -inline __device__ Float4_ fma(uint16_t a, uint2 b, Float4_ fc)
443 -{
444 - uint32_t s = h0_h0(a);
445 - Float4_ fd;
446 - fd.x = fma(s, b.x, fc.x);
447 - fd.y = fma(s, b.y, fc.y);
448 - return fd;
449 -}
450 -
451 -////////////////////////////////////////////////////////////////////////////////////////////////////
452 -
453 -inline __device__ Float8_ fma(uint4 a, uint4 b, Float8_ fc)
454 -{
455 - Float8_ fd;
456 - fd.x = fma(a.x, b.x, fc.x);
457 - fd.y = fma(a.y, b.y, fc.y);
458 - fd.z = fma(a.z, b.z, fc.z);
459 - fd.w = fma(a.w, b.w, fc.w);
460 - return fd;
461 -}
462 -
463 -////////////////////////////////////////////////////////////////////////////////////////////////////
464 -
465 -inline __device__ Float8_ fma(uint16_t a, uint4 b, Float8_ fc)
466 -{
467 - uint32_t s = h0_h0(a);
468 - Float8_ fd;
469 - fd.x = fma(s, b.x, fc.x);
470 - fd.y = fma(s, b.y, fc.y);
471 - fd.z = fma(s, b.z, fc.z);
472 - fd.w = fma(s, b.w, fc.w);
473 - return fd;
474 -}
475 -
476 -////////////////////////////////////////////////////////////////////////////////////////////////////
477 -#ifdef ENABLE_BF16
478 -inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b, __nv_bfloat162 c)
479 -{
480 - return bf16hfma2(a, b, c);
481 -}
482 -
483 -////////////////////////////////////////////////////////////////////////////////////////////////////
484 -
485 -inline __device__ __nv_bfloat162 fma(__nv_bfloat16 a, __nv_bfloat162 b, __nv_bfloat162 c)
486 -{
487 - return bf16hfma2(bf162bf162(a), b, c);
488 -}
489 -////////////////////////////////////////////////////////////////////////////////////////////////////
490 -
491 -inline __device__ bf16_4_t fma(bf16_4_t a, bf16_4_t b, bf16_4_t c)
492 -{
493 - bf16_4_t d;
494 - d.x = fma(a.x, b.x, c.x);
495 - d.y = fma(a.y, b.y, c.y);
496 - return d;
497 -}
498 -
499 -////////////////////////////////////////////////////////////////////////////////////////////////////
500 -
501 -inline __device__ bf16_4_t fma(__nv_bfloat16 a, bf16_4_t b, bf16_4_t c)
502 -{
503 - __nv_bfloat162 s = bf162bf162(a);
504 - bf16_4_t d;
505 - d.x = fma(s, b.x, c.x);
506 - d.y = fma(s, b.y, c.y);
507 - return d;
508 -}
509 -
510 -////////////////////////////////////////////////////////////////////////////////////////////////////
511 -
512 -inline __device__ bf16_8_t fma(bf16_8_t a, bf16_8_t b, bf16_8_t c)
513 -{
514 - bf16_8_t d;
515 - d.x = fma(a.x, b.x, c.x);
516 - d.y = fma(a.y, b.y, c.y);
517 - d.z = fma(a.z, b.z, c.z);
518 - d.w = fma(a.w, b.w, c.w);
519 - return d;
520 -}
521 -
522 -////////////////////////////////////////////////////////////////////////////////////////////////////
523 -
524 -inline __device__ bf16_8_t fma(__nv_bfloat16 a, bf16_8_t b, bf16_8_t c)
525 -{
526 - __nv_bfloat162 s = bf162bf162(a);
527 - bf16_8_t d;
528 - d.x = fma(s, b.x, c.x);
529 - d.y = fma(s, b.y, c.y);
530 - d.z = fma(s, b.z, c.z);
531 - d.w = fma(s, b.w, c.w);
532 - return d;
533 -}
534 -
535 -////////////////////////////////////////////////////////////////////////////////////////////////////
536 -
537 -inline __device__ float fma(__nv_bfloat16 a, __nv_bfloat16 b, float fc)
538 -{
539 - return __bfloat162float(a) * __bfloat162float(b) + fc;
540 -}
541 -
542 -////////////////////////////////////////////////////////////////////////////////////////////////////
543 -
544 -inline __device__ float2 fma(__nv_bfloat162 a, __nv_bfloat162 b, float2 fc)
545 -{
546 - float2 fa = bf1622float2(a);
547 - float2 fb = bf1622float2(b);
548 - return fma(fa, fb, fc);
549 -}
550 -
551 -////////////////////////////////////////////////////////////////////////////////////////////////////
552 -
553 -inline __device__ float2 fma(__nv_bfloat16 a, __nv_bfloat162 b, float2 fc)
554 -{
555 - return fma(bf162bf162(a), b, fc);
556 -}
557 -
558 -////////////////////////////////////////////////////////////////////////////////////////////////////
559 -
560 -inline __device__ Float4_ fma(bf16_4_t a, bf16_4_t b, Float4_ fc)
561 -{
562 - Float4_ fd;
563 - fd.x = fma(a.x, b.x, fc.x);
564 - fd.y = fma(a.y, b.y, fc.y);
565 - return fd;
566 -}
567 -
568 -////////////////////////////////////////////////////////////////////////////////////////////////////
569 -
570 -inline __device__ Float4_ fma(__nv_bfloat16 a, bf16_4_t b, Float4_ fc)
571 -{
572 - __nv_bfloat162 s = bf162bf162(a);
573 - Float4_ fd;
574 - fd.x = fma(s, b.x, fc.x);
575 - fd.y = fma(s, b.y, fc.y);
576 - return fd;
577 -}
578 -
579 -////////////////////////////////////////////////////////////////////////////////////////////////////
580 -
581 -inline __device__ Float8_ fma(bf16_8_t a, bf16_8_t b, Float8_ fc)
582 -{
583 - Float8_ fd;
584 - fd.x = fma(a.x, b.x, fc.x);
585 - fd.y = fma(a.y, b.y, fc.y);
586 - fd.z = fma(a.z, b.z, fc.z);
587 - fd.w = fma(a.w, b.w, fc.w);
588 - return fd;
589 -}
590 -
591 -////////////////////////////////////////////////////////////////////////////////////////////////////
592 -
593 -inline __device__ Float8_ fma(__nv_bfloat16 a, bf16_8_t b, Float8_ fc)
594 -{
595 - __nv_bfloat162 s = bf162bf162(a);
596 - Float8_ fd;
597 - fd.x = fma(s, b.x, fc.x);
598 - fd.y = fma(s, b.y, fc.y);
599 - fd.z = fma(s, b.z, fc.z);
600 - fd.w = fma(s, b.w, fc.w);
601 - return fd;
602 -}
603 -#endif // ENABLE_BF16
604 -////////////////////////////////////////////////////////////////////////////////////////////////////
605 -
606 -template<typename Acc, typename A, typename B>
607 -inline __device__ Acc mul(A a, B b);
608 -
609 -////////////////////////////////////////////////////////////////////////////////////////////////////
610 -
611 -template<>
612 -inline __device__ float mul<float, float>(float a, float b)
613 -{
614 - return a * b;
615 -}
616 -
617 -////////////////////////////////////////////////////////////////////////////////////////////////////
618 -
619 -template<>
620 -inline __device__ float2 mul(float2 a, float2 b)
621 -{
622 - float2 c;
623 - c.x = a.x * b.x;
624 - c.y = a.y * b.y;
625 - return c;
626 -}
627 -
628 -////////////////////////////////////////////////////////////////////////////////////////////////////
629 -
630 -template<>
631 -inline __device__ float2 mul(float a, float2 b)
632 -{
633 - float2 c;
634 - c.x = a * b.x;
635 - c.y = a * b.y;
636 - return c;
637 -}
638 -
639 -////////////////////////////////////////////////////////////////////////////////////////////////////
640 -
641 -template<>
642 -inline __device__ float4 mul(float4 a, float4 b)
643 -{
644 - float4 c;
645 - c.x = a.x * b.x;
646 - c.y = a.y * b.y;
647 - c.z = a.z * b.z;
648 - c.w = a.w * b.w;
649 - return c;
650 -}
651 -
652 -////////////////////////////////////////////////////////////////////////////////////////////////////
653 -
654 -template<>
655 -inline __device__ float4 mul(float a, float4 b)
656 -{
657 - float4 c;
658 - c.x = a * b.x;
659 - c.y = a * b.y;
660 - c.z = a * b.z;
661 - c.w = a * b.w;
662 - return c;
663 -}
664 -
665 -////////////////////////////////////////////////////////////////////////////////////////////////////
666 -
667 -template<>
668 -inline __device__ uint16_t mul(uint16_t a, uint16_t b)
669 -{
670 - uint16_t c;
671 - asm volatile("mul.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
672 - return c;
673 -}
674 -
675 -////////////////////////////////////////////////////////////////////////////////////////////////////
676 -
677 -template<>
678 -inline __device__ uint32_t mul(uint32_t a, uint32_t b)
679 -{
680 - uint32_t c;
681 - asm volatile("mul.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
682 - return c;
683 -}
684 -
685 -////////////////////////////////////////////////////////////////////////////////////////////////////
686 -
687 -template<>
688 -inline __device__ uint32_t mul(uint16_t a, uint32_t b)
689 -{
690 - return mul<uint32_t, uint32_t, uint32_t>(h0_h0(a), b);
691 -}
692 -
693 -////////////////////////////////////////////////////////////////////////////////////////////////////
694 -
695 -template<>
696 -inline __device__ uint2 mul(uint2 a, uint2 b)
697 -{
698 - uint2 c;
699 - c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
700 - c.y = mul<uint32_t, uint32_t, uint32_t>(a.y, b.y);
701 - return c;
702 -}
703 -
704 -////////////////////////////////////////////////////////////////////////////////////////////////////
705 -
706 -template<>
707 -inline __device__ uint2 mul(uint16_t a, uint2 b)
708 -{
709 - uint32_t s = h0_h0(a);
710 - uint2 c;
711 - c.x = mul<uint32_t, uint32_t, uint32_t>(s, b.x);
712 - c.y = mul<uint32_t, uint32_t, uint32_t>(s, b.y);
713 - return c;
714 -}
715 -
716 -////////////////////////////////////////////////////////////////////////////////////////////////////
717 -
718 -template<>
719 -inline __device__ uint4 mul(uint4 a, uint4 b)
720 -{
721 - uint4 c;
722 - c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
723 - c.y = mul<uint32_t, uint32_t, uint32_t>(a.y, b.y);
724 - c.z = mul<uint32_t, uint32_t, uint32_t>(a.z, b.z);
725 - c.w = mul<uint32_t, uint32_t, uint32_t>(a.w, b.w);
726 - return c;
727 -}
728 -
729 -////////////////////////////////////////////////////////////////////////////////////////////////////
730 -
731 -template<>
732 -inline __device__ uint4 mul(uint16_t a, uint4 b)
733 -{
734 - uint32_t s = h0_h0(a);
735 - uint4 c;
736 - c.x = mul<uint32_t, uint32_t, uint32_t>(s, b.x);
737 - c.y = mul<uint32_t, uint32_t, uint32_t>(s, b.y);
738 - c.z = mul<uint32_t, uint32_t, uint32_t>(s, b.z);
739 - c.w = mul<uint32_t, uint32_t, uint32_t>(s, b.w);
740 - return c;
741 -}
742 -
743 -////////////////////////////////////////////////////////////////////////////////////////////////////
744 -
745 -template<>
746 -inline __device__ float mul(uint16_t a, uint16_t b)
747 -{
748 - float fa = half_to_float(a);
749 - float fb = half_to_float(b);
750 - return fa * fb;
751 -}
752 -
753 -////////////////////////////////////////////////////////////////////////////////////////////////////
754 -
755 -template<>
756 -inline __device__ float2 mul(uint32_t a, uint32_t b)
757 -{
758 - float2 fa = half2_to_float2(a);
759 - float2 fb = half2_to_float2(b);
760 - return mul<float2, float2, float2>(fa, fb);
761 -}
762 -
763 -////////////////////////////////////////////////////////////////////////////////////////////////////
764 -
765 -template<>
766 -inline __device__ float2 mul(uint16_t a, uint32_t b)
767 -{
768 - return mul<float2, uint32_t, uint32_t>(h0_h0(a), b);
769 -}
770 -
771 -////////////////////////////////////////////////////////////////////////////////////////////////////
772 -
773 -template<>
774 -inline __device__ Float4_ mul(uint2 a, uint2 b)
775 -{
776 - Float4_ fc;
777 - fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
778 - fc.y = mul<float2, uint32_t, uint32_t>(a.y, b.y);
779 - return fc;
780 -}
781 -
782 -////////////////////////////////////////////////////////////////////////////////////////////////////
783 -
784 -template<>
785 -inline __device__ Float4_ mul(uint16_t a, uint2 b)
786 -{
787 - uint32_t s = h0_h0(a);
788 - Float4_ fc;
789 - fc.x = mul<float2, uint32_t, uint32_t>(s, b.x);
790 - fc.y = mul<float2, uint32_t, uint32_t>(s, b.y);
791 - return fc;
792 -}
793 -
794 -////////////////////////////////////////////////////////////////////////////////////////////////////
795 -
796 -template<>
797 -inline __device__ Float8_ mul(uint4 a, uint4 b)
798 -{
799 - Float8_ fc;
800 - fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
801 - fc.y = mul<float2, uint32_t, uint32_t>(a.y, b.y);
802 - fc.z = mul<float2, uint32_t, uint32_t>(a.z, b.z);
803 - fc.w = mul<float2, uint32_t, uint32_t>(a.w, b.w);
804 - return fc;
805 -}
806 -
807 -////////////////////////////////////////////////////////////////////////////////////////////////////
808 -
809 -template<>
810 -inline __device__ Float8_ mul(uint16_t a, uint4 b)
811 -{
812 - uint32_t s = h0_h0(a);
813 - Float8_ fc;
814 - fc.x = mul<float2, uint32_t, uint32_t>(s, b.x);
815 - fc.y = mul<float2, uint32_t, uint32_t>(s, b.y);
816 - fc.z = mul<float2, uint32_t, uint32_t>(s, b.z);
817 - fc.w = mul<float2, uint32_t, uint32_t>(s, b.w);
818 - return fc;
819 -}
820 -
821 -////////////////////////////////////////////////////////////////////////////////////////////////////
822 -
823 -#ifdef ENABLE_BF16
824 -template<>
825 -inline __device__ __nv_bfloat16 mul(__nv_bfloat16 a, __nv_bfloat16 b)
826 -{
827 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
828 - return __hmul(a, b);
829 -#else
830 - return bf16hmul(a, b);
831 -#endif
832 -}
833 -
834 -////////////////////////////////////////////////////////////////////////////////////////////////////
835 -
836 -template<>
837 -inline __device__ __nv_bfloat162 mul(__nv_bfloat162 a, __nv_bfloat162 b)
838 -{
839 - return bf16hmul2(a, b);
840 -}
841 -
842 -////////////////////////////////////////////////////////////////////////////////////////////////////
843 -
844 -template<>
845 -inline __device__ __nv_bfloat162 mul(__nv_bfloat16 a, __nv_bfloat162 b)
846 -{
847 - return mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
848 -}
849 -
850 -////////////////////////////////////////////////////////////////////////////////////////////////////
851 -
852 -template<>
853 -inline __device__ bf16_4_t mul(bf16_4_t a, bf16_4_t b)
854 -{
855 - bf16_4_t c;
856 - c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
857 - c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
858 - return c;
859 -}
860 -
861 -////////////////////////////////////////////////////////////////////////////////////////////////////
862 -
863 -template<>
864 -inline __device__ bf16_4_t mul(__nv_bfloat16 a, bf16_4_t b)
865 -{
866 - __nv_bfloat162 s = bf162bf162(a);
867 - bf16_4_t c;
868 - c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.x);
869 - c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.y);
870 - return c;
871 -}
872 -
873 -////////////////////////////////////////////////////////////////////////////////////////////////////
874 -
875 -template<>
876 -inline __device__ bf16_8_t mul(bf16_8_t a, bf16_8_t b)
877 -{
878 - bf16_8_t c;
879 - c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
880 - c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
881 - c.z = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.z, b.z);
882 - c.w = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.w, b.w);
883 - return c;
884 -}
885 -
886 -////////////////////////////////////////////////////////////////////////////////////////////////////
887 -
888 -template<>
889 -inline __device__ bf16_8_t mul(__nv_bfloat16 a, bf16_8_t b)
890 -{
891 - __nv_bfloat162 s = bf162bf162(a);
892 - bf16_8_t c;
893 - c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.x);
894 - c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.y);
895 - c.z = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.z);
896 - c.w = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.w);
897 - return c;
898 -}
899 -
900 -////////////////////////////////////////////////////////////////////////////////////////////////////
901 -
902 -template<>
903 -inline __device__ float mul(__nv_bfloat16 a, __nv_bfloat16 b)
904 -{
905 - float fa = (float)a;
906 - float fb = (float)b;
907 - return fa * fb;
908 -}
909 -
910 -////////////////////////////////////////////////////////////////////////////////////////////////////
911 -
912 -template<>
913 -inline __device__ float2 mul(__nv_bfloat162 a, __nv_bfloat162 b)
914 -{
915 - float2 fa = bf1622float2(a);
916 - float2 fb = bf1622float2(b);
917 - return mul<float2, float2, float2>(fa, fb);
918 -}
919 -
920 -////////////////////////////////////////////////////////////////////////////////////////////////////
921 -
922 -template<>
923 -inline __device__ float2 mul(__nv_bfloat16 a, __nv_bfloat162 b)
924 -{
925 - return mul<float2, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
926 -}
927 -
928 -////////////////////////////////////////////////////////////////////////////////////////////////////
929 -
930 -template<>
931 -inline __device__ Float4_ mul(bf16_4_t a, bf16_4_t b)
932 -{
933 - Float4_ fc;
934 - fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
935 - fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
936 - return fc;
937 -}
938 -
939 -////////////////////////////////////////////////////////////////////////////////////////////////////
940 -
941 -template<>
942 -inline __device__ Float4_ mul(__nv_bfloat16 a, bf16_4_t b)
943 -{
944 - __nv_bfloat162 s = bf162bf162(a);
945 - Float4_ fc;
946 - fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.x);
947 - fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.y);
948 - return fc;
949 -}
950 -
951 -////////////////////////////////////////////////////////////////////////////////////////////////////
952 -
953 -template<>
954 -inline __device__ Float8_ mul(bf16_8_t a, bf16_8_t b)
955 -{
956 - Float8_ fc;
957 - fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
958 - fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
959 - fc.z = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.z, b.z);
960 - fc.w = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.w, b.w);
961 - return fc;
962 -}
963 -
964 -////////////////////////////////////////////////////////////////////////////////////////////////////
965 -
966 -template<>
967 -inline __device__ Float8_ mul(__nv_bfloat16 a, bf16_8_t b)
968 -{
969 - __nv_bfloat162 s = bf162bf162(a);
970 - Float8_ fc;
971 - fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.x);
972 - fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.y);
973 - fc.z = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.z);
974 - fc.w = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.w);
975 - return fc;
976 -}
977 -#endif // ENABLE_BF16
978 -////////////////////////////////////////////////////////////////////////////////////////////////////
979 -
980 -inline __device__ float sum(float v)
981 -{
982 - return v;
983 -}
984 -
985 -////////////////////////////////////////////////////////////////////////////////////////////////////
986 -
987 -inline __device__ float sum(float2 v)
988 -{
989 - return v.x + v.y;
990 -}
991 -
992 -////////////////////////////////////////////////////////////////////////////////////////////////////
993 -
994 -inline __device__ float sum(float4 v)
995 -{
996 - return v.x + v.y + v.z + v.w;
997 -}
998 -
999 -////////////////////////////////////////////////////////////////////////////////////////////////////
1000 -
1001 -#ifdef ENABLE_BF16
1002 -inline __device__ float sum(__nv_bfloat162 v)
1003 -{
1004 - float2 vf = bf1622float2(v);
1005 - return vf.x + vf.y;
1006 -}
1007 -
1008 -////////////////////////////////////////////////////////////////////////////////////////////////////
1009 -
1010 -inline __device__ float sum(bf16_4_t v)
1011 -{
1012 - return sum(v.x) + sum(v.y);
1013 -}
1014 -
1015 -////////////////////////////////////////////////////////////////////////////////////////////////////
1016 -
1017 -inline __device__ float sum(bf16_8_t v)
1018 -{
1019 - return sum(v.x) + sum(v.y) + sum(v.z) + sum(v.w);
1020 -}
1021 -#endif // ENABLE_BF16
1022 -////////////////////////////////////////////////////////////////////////////////////////////////////
1023 -
1024 -inline __device__ float sum(uint16_t v)
1025 -{
1026 - return half_to_float(v);
1027 -}
1028 -
1029 -////////////////////////////////////////////////////////////////////////////////////////////////////
1030 -
1031 -inline __device__ float sum(uint32_t v)
1032 -{
1033 - float2 tmp = half2_to_float2(v);
1034 - return tmp.x + tmp.y;
1035 -}
1036 -
1037 -////////////////////////////////////////////////////////////////////////////////////////////////////
1038 -
1039 -inline __device__ float sum(uint2 v)
1040 -{
1041 - uint32_t c = add(v.x, v.y);
1042 - return sum(c);
1043 -}
1044 -
1045 -////////////////////////////////////////////////////////////////////////////////////////////////////
1046 -
1047 -inline __device__ float sum(uint4 v)
1048 -{
1049 -#if 1
1050 - uint32_t c = add(v.x, v.y);
1051 - c = add(c, v.z);
1052 - c = add(c, v.w);
1053 -#else
1054 - uint32_t c = add(v.x, v.y);
1055 - uint32_t d = add(v.z, v.w);
1056 - c = add(c, d);
1057 -#endif
1058 - return sum(c);
1059 -}
1060 -
1061 -////////////////////////////////////////////////////////////////////////////////////////////////////
1062 -
1063 -inline __device__ float sum(Float4_ v)
1064 -{
1065 - return v.x.x + v.x.y + v.y.x + v.y.y;
1066 -}
1067 -
1068 -////////////////////////////////////////////////////////////////////////////////////////////////////
1069 -
1070 -inline __device__ float sum(Float8_ v)
1071 -{
1072 - return v.x.x + v.x.y + v.y.x + v.y.y + v.z.x + v.z.y + v.w.x + v.w.y;
1073 -}
1074 -
1075 -////////////////////////////////////////////////////////////////////////////////////////////////////
1076 -
1077 -inline __device__ float dot(float a, float b)
1078 -{
1079 - return a * b;
1080 -}
1081 -
1082 -////////////////////////////////////////////////////////////////////////////////////////////////////
1083 -
1084 -inline __device__ float dot(float2 a, float2 b)
1085 -{
1086 - float2 c = mul<float2, float2, float2>(a, b);
1087 - return c.x + c.y;
1088 -}
1089 -
1090 -////////////////////////////////////////////////////////////////////////////////////////////////////
1091 -
1092 -inline __device__ float dot(Float4_ a, Float4_ b)
1093 -{
1094 - float2 acc = mul<float2, float2, float2>(a.x, b.x);
1095 - acc = fma(a.y, b.y, acc);
1096 - return acc.x + acc.y;
1097 -}
1098 -
1099 -////////////////////////////////////////////////////////////////////////////////////////////////////
1100 -
1101 -inline __device__ float dot(Float8_ a, Float8_ b)
1102 -{
1103 - float2 acc = mul<float2, float2, float2>(a.x, b.x);
1104 - acc = fma(a.y, b.y, acc);
1105 - acc = fma(a.z, b.z, acc);
1106 - acc = fma(a.w, b.w, acc);
1107 - return acc.x + acc.y;
1108 -}
1109 -
1110 -////////////////////////////////////////////////////////////////////////////////////////////////////
1111 -
1112 -template<typename T>
1113 -inline __device__ float dot(T a, T b)
1114 -{
1115 - return sum(mul<T, T, T>(a, b));
1116 -}
1117 -
1118 -////////////////////////////////////////////////////////////////////////////////////////////////////
1119 -
1120 -template<typename A, typename T>
1121 -inline __device__ float dot(T a, T b)
1122 -{
1123 - return sum(mul<A, T, T>(a, b));
1124 -}
1125 -
1126 -////////////////////////////////////////////////////////////////////////////////////////////////////
1127 -
1128 -inline __device__ void zero(uint16_t& dst)
1129 -{
1130 - dst = uint16_t(0);
1131 -}
1132 -
1133 -////////////////////////////////////////////////////////////////////////////////////////////////////
1134 -
1135 -template<typename T>
1136 -inline __device__ void zero(T& dst)
1137 -{
1138 - constexpr int WORDS = sizeof(T) / 4;
1139 - union {
1140 - T raw;
1141 - uint32_t words[WORDS];
1142 - } tmp;
1143 -#pragma unroll
1144 - for (int ii = 0; ii < WORDS; ++ii) {
1145 - tmp.words[ii] = 0u;
1146 - }
1147 - dst = tmp.raw;
1148 -}
1149 -
1150 -
1151 -////////////////////////////////////////////////////////////////////////////////////////////////////
1152 -
1153 -inline __device__ void convert_from_float(float& dst, float src)
1154 -{
1155 - dst = src;
1156 -}
1157 -
1158 -////////////////////////////////////////////////////////////////////////////////////////////////////
1159 -
1160 -inline __device__ void convert_from_float(uint16_t& dst, float src)
1161 -{
1162 - dst = float_to_half(src);
1163 -}
1164 -
1165 -////////////////////////////////////////////////////////////////////////////////////////////////////
1166 -
1167 -inline __device__ void convert_from_float(uint32_t& dst, float2 src)
1168 -{
1169 - dst = float2_to_half2(src);
1170 -}
1171 -
1172 -////////////////////////////////////////////////////////////////////////////////////////////////////
1173 -#ifdef ENABLE_BF16
1174 -inline __device__ void convert_from_float(__nv_bfloat16& dst, float src)
1175 -{
1176 - dst = __float2bfloat16(src);
1177 -}
1178 -
1179 -////////////////////////////////////////////////////////////////////////////////////////////////////
1180 -
1181 -inline __device__ void convert_from_float(__nv_bfloat162& dst, float2 src)
1182 -{
1183 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1184 - dst = __float22bfloat162_rn(src);
1185 -#else
1186 - dst = __floats2bfloat162_rn(src.x, src.y);
1187 -#endif
1188 -}
1189 -#endif // ENABLE_BF16
1190 -////////////////////////////////////////////////////////////////////////////////////////////////////
1191 -
1192 -inline __device__ void convert_from_float(uint2& dst, Float4_ src)
1193 -{
1194 - dst.x = float2_to_half2(src.x);
1195 - dst.y = float2_to_half2(src.y);
1196 -}
1197 -
1198 -////////////////////////////////////////////////////////////////////////////////////////////////////
1199 -
1200 -inline __device__ void convert_from_float(uint4& dst, Float8_ src)
1201 -{
1202 - dst.x = float2_to_half2(src.x);
1203 - dst.y = float2_to_half2(src.y);
1204 - dst.z = float2_to_half2(src.z);
1205 - dst.w = float2_to_half2(src.w);
1206 -}
1207 -
1208 -////////////////////////////////////////////////////////////////////////////////////////////////////
1209 -
1210 -#ifdef ENABLE_BF16
1211 -inline __device__ void convert_from_float(bf16_4_t& dst, Float4_ src)
1212 -{
1213 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1214 - dst.x = __float22bfloat162_rn(src.x);
1215 - dst.y = __float22bfloat162_rn(src.y);
1216 -#else
1217 - dst.x = __floats2bfloat162_rn(src.x.x, src.x.y);
1218 - dst.y = __floats2bfloat162_rn(src.y.x, src.y.y);
1219 -#endif
1220 -}
1221 -
1222 -////////////////////////////////////////////////////////////////////////////////////////////////////
1223 -
1224 -inline __device__ void convert_from_float(bf16_8_t& dst, Float8_ src)
1225 -{
1226 -#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1227 - dst.x = __float22bfloat162_rn(src.x);
1228 - dst.y = __float22bfloat162_rn(src.y);
1229 - dst.z = __float22bfloat162_rn(src.z);
1230 - dst.w = __float22bfloat162_rn(src.w);
1231 -#else
1232 - dst.x = __floats2bfloat162_rn(src.x.x, src.x.y);
1233 - dst.y = __floats2bfloat162_rn(src.y.x, src.y.y);
1234 - dst.z = __floats2bfloat162_rn(src.z.x, src.z.y);
1235 - dst.w = __floats2bfloat162_rn(src.w.x, src.w.y);
1236 -#endif
1237 -}
1238 -#endif // ENABLE_BF16
1239 -
1240 -////////////////////////////////////////////////////////////////////////////////////////////////////
1241 -
1242 -inline __device__ void convert_from_float(float2& dst, float2 src)
1243 -{
1244 - dst = src;
1245 -}
1246 -
1247 -////////////////////////////////////////////////////////////////////////////////////////////////////
1248 -
1249 -inline __device__ void convert_from_float(float4& dst, float4 src)
1250 -{
1251 - dst = src;
1252 -}
1253 -
1254 -////////////////////////////////////////////////////////////////////////////////////////////////////
1255 -
1256 -inline __device__ float convert_to_float(float4 u)
1257 -{
1258 - return u.x;
1259 -}
1260 -
1261 -////////////////////////////////////////////////////////////////////////////////////////////////////
1262 -
1263 -inline __device__ float convert_to_float(uint4 u)
1264 -{
1265 - float2 tmp = half2_to_float2(u.x);
1266 - return tmp.x;
1267 -}
1268 -
1269 -////////////////////////////////////////////////////////////////////////////////////////////////////
1270 -
1271 -// inline __device__ float cast_to_float(float u)
1272 -// {
1273 -// return u;
1274 -// }
1275 -
1276 -////////////////////////////////////////////////////////////////////////////////////////////////////
1277 -
1278 -// inline __device__ float2 cast_to_float(float2 u)
1279 -// {
1280 -// return u;
1281 -// }
1282 -
1283 -////////////////////////////////////////////////////////////////////////////////////////////////////
1284 -
1285 -// inline __device__ float4 cast_to_float(float4 u)
1286 -// {
1287 -// return u;
1288 -// }
1289 -
1290 -////////////////////////////////////////////////////////////////////////////////////////////////////
1291 -
1292 -// inline __device__ Float4_ cast_to_float(Float4_ u)
1293 -// {
1294 -// return u;
1295 -// }
1296 -
1297 -////////////////////////////////////////////////////////////////////////////////////////////////////
1298 -
1299 -// inline __device__ Float8_ cast_to_float(Float8_ u)
1300 -// {
1301 -// return u;
1302 -// }
1303 -
1304 -////////////////////////////////////////////////////////////////////////////////////////////////////
1305 -
1306 -inline __device__ float cast_to_float(uint16_t u)
1307 -{
1308 - return half_to_float(u);
1309 -}
1310 -
1311 -////////////////////////////////////////////////////////////////////////////////////////////////////
1312 -
1313 -inline __device__ float2 cast_to_float(uint32_t u)
1314 -{
1315 - return half2_to_float2(u);
1316 -}
1317 -
1318 -////////////////////////////////////////////////////////////////////////////////////////////////////
1319 -
1320 -inline __device__ Float4_ cast_to_float(uint2 u)
1321 -{
1322 - Float4_ tmp;
1323 - tmp.x = half2_to_float2(u.x);
1324 - tmp.y = half2_to_float2(u.y);
1325 - return tmp;
1326 -}
1327 -
1328 -////////////////////////////////////////////////////////////////////////////////////////////////////
1329 -
1330 -inline __device__ Float8_ cast_to_float(uint4 u)
1331 -{
1332 - Float8_ tmp;
1333 - tmp.x = half2_to_float2(u.x);
1334 - tmp.y = half2_to_float2(u.y);
1335 - tmp.z = half2_to_float2(u.z);
1336 - tmp.w = half2_to_float2(u.w);
1337 - return tmp;
1338 -}
1339 -
1340 -}
csrc/layernorm_kernels.cu
+1 -1
@@ -1,7 +1,7 @@
1 #include <torch/extension.h>
2 #include <ATen/cuda/CUDAContext.h>
3
4 -#include "reduction_utils.h"
4 +#include "reduction_utils.cuh"
5
6 namespace cacheflow {
7
csrc/reduction_utils.cuh new
+34
@@ -0,0 +1,34 @@
1 +#pragma once
2 +
3 +namespace cacheflow {
4 +
5 +template<typename T>
6 +__inline__ __device__ T warpReduceSum(T val) {
7 +#pragma unroll
8 + for (int mask = 16; mask > 0; mask >>= 1)
9 + val += __shfl_xor_sync(0xffffffff, val, mask, 32);
10 + return val;
11 +}
12 +
13 +/* Calculate the sum of all elements in a block */
14 +template<typename T>
15 +__inline__ __device__ T blockReduceSum(T val) {
16 + static __shared__ T shared[32];
17 + int lane = threadIdx.x & 0x1f;
18 + int wid = threadIdx.x >> 5;
19 +
20 + val = warpReduceSum<T>(val);
21 +
22 + if (lane == 0)
23 + shared[wid] = val;
24 +
25 + __syncthreads();
26 +
27 + // Modify from blockDim.x << 5 to blockDim.x / 32. to prevent
28 + // blockDim.x is not divided by 32
29 + val = (threadIdx.x < (blockDim.x / 32.f)) ? shared[lane] : (T)(0.0f);
30 + val = warpReduceSum<T>(val);
31 + return val;
32 +}
33 +
34 +} // namespace cacheflow
csrc/reduction_utils.h deleted
-76
@@ -1,76 +0,0 @@
1 -#pragma once
2 -
3 -namespace cacheflow {
4 -
5 -template<int WARPS_PER_BLOCK, int WARP_SIZE = 32>
6 -inline __device__ float block_sum(float* red_smem, float sum)
7 -{
8 -
9 - // Decompose the thread index into warp / lane.
10 - int warp = threadIdx.x / WARP_SIZE;
11 - int lane = threadIdx.x % WARP_SIZE;
12 -
13 - // Compute the sum per warp.
14 -#pragma unroll
15 - for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
16 - sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
17 - }
18 -
19 - // Warp leaders store the data to shared memory.
20 - if (lane == 0) {
21 - red_smem[warp] = sum;
22 - }
23 -
24 - // Make sure the data is in shared memory.
25 - __syncthreads();
26 -
27 - // The warps compute the final sums.
28 - if (lane < WARPS_PER_BLOCK) {
29 - sum = red_smem[lane];
30 - }
31 -
32 -// Parallel reduction inside the warp.
33 -#pragma unroll
34 - for (int mask = WARPS_PER_BLOCK / 2; mask >= 1; mask /= 2) {
35 - sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
36 - }
37 -
38 - // Broadcast to other threads.
39 - return __shfl_sync(uint32_t(-1), sum, 0);
40 -}
41 -
42 -#define FINAL_MASK 0xffffffff
43 -
44 -template<typename T>
45 -__inline__ __device__ T warpReduceSum(T val)
46 -{
47 -#pragma unroll
48 - for (int mask = 16; mask > 0; mask >>= 1)
49 - val += __shfl_xor_sync(FINAL_MASK, val, mask, 32);
50 - return val;
51 -}
52 -
53 -/* Calculate the sum of all elements in a block */
54 -template<typename T>
55 -__inline__ __device__ T blockReduceSum(T val)
56 -{
57 - static __shared__ T shared[32];
58 - int lane = threadIdx.x & 0x1f;
59 - int wid = threadIdx.x >> 5;
60 -
61 - val = warpReduceSum<T>(val);
62 -
63 - if (lane == 0)
64 - shared[wid] = val;
65 -
66 - __syncthreads();
67 -
68 - // Modify from blockDim.x << 5 to blockDim.x / 32. to prevent
69 - // blockDim.x is not divided by 32
70 - val = (threadIdx.x < (blockDim.x / 32.f)) ? shared[lane] : (T)(0.0f);
71 - val = warpReduceSum<T>(val);
72 -
73 - return val;
74 -}
75 -
76 -} // namespace cacheflow
setup.py
+1 -1
@@ -18,7 +18,7 @@ ext_modules.append(cache_extension)
18 # Attention kernels.
19 attention_extension = cpp_extension.CUDAExtension(
20 name='cacheflow.attention_ops',
21 - sources=['csrc/attention.cpp', 'csrc/attention_kernels.cu'],
21 + sources=['csrc/attention.cpp', 'csrc/attention/attention_kernels.cu'],
22 extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
23 )
24 ext_modules.append(attention_extension)
tests/kernels/attention.py
-90
@@ -271,78 +271,6 @@ def test_multi_query_kv_attention(
271 assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
272
273
274 -def test_multi_query_cached_kv_attention(
275 - num_queries: int,
276 - num_heads: int,
277 - head_size: int,
278 - block_size: int,
279 - num_blocks: int,
280 - dtype: torch.dtype,
281 -) -> None:
282 - query_lens = random.sample(range(1, MAX_SEQ_LEN), num_queries)
283 - cu_query_lens = [0]
284 - for query_len in query_lens:
285 - cu_query_lens.append(cu_query_lens[-1] + query_len)
286 - num_total_tokens = cu_query_lens[-1]
287 -
288 - qkv = torch.randn(
289 - num_total_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
290 - query, _, _ = qkv.unbind(dim=1)
291 - x = 16 // torch.tensor([], dtype=dtype).element_size()
292 - key_block_shape = (num_heads, head_size // x, block_size, x)
293 - key_cache = torch.randn(
294 - size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
295 - value_block_shape = (num_heads, head_size, block_size)
296 - value_cache = torch.randn(
297 - size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
298 -
299 - cu_query_lens = torch.tensor(cu_query_lens, dtype=torch.int, device='cuda')
300 - context_lens = [
301 - query_len + random.randint(0, MAX_SEQ_LEN - query_len)
302 - for query_len in query_lens
303 - ]
304 - max_context_len = max(context_lens)
305 - context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
306 -
307 - max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
308 - block_tables = []
309 - for _ in range(num_queries):
310 - block_table = [
311 - random.randint(0, num_blocks - 1)
312 - for _ in range(max_num_blocks_per_seq)
313 - ]
314 - block_tables.append(block_table)
315 - block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
316 -
317 - scale = float(1.0 / (head_size ** 0.5))
318 - output = torch.empty(
319 - num_total_tokens, num_heads, head_size, dtype=dtype, device='cuda')
320 -
321 - attention_ops.multi_query_cached_kv_attention(
322 - cu_query_lens,
323 - output,
324 - query,
325 - key_cache,
326 - value_cache,
327 - scale,
328 - block_tables,
329 - context_lens,
330 - block_size,
331 - max_context_len,
332 - )
333 -
334 - ref_output = ref_multi_query_cached_kv_attention(
335 - cu_query_lens,
336 - query,
337 - key_cache,
338 - value_cache,
339 - block_tables,
340 - context_lens,
341 - dtype,
342 - )
343 - assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
344 -
345 -
274 @torch.inference_mode()
275 def test_attention(seed: int) -> None:
276 # NOTE(woosuk): Even when the seed is fixed, there is a chance that
@@ -364,24 +292,6 @@ def test_attention(seed: int) -> None:
292 dtype=dtype,
293 )
294
367 - # NOTE(siyuan): Same as above. Re-run the test if it fails. Also
368 - # note that the test is also more likely to fail due to the much
369 - # larger amount of tokens in the input may increase the variance.
370 - for dtype in [torch.half, torch.float]:
371 - for block_size in [8, 16, 32]:
372 - for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
373 - print(f'Testing multi_query_cached_kv_attention with '
374 - f'dtype={dtype}, block_size={block_size}, '
375 - f'head_size={head_size}')
376 - test_multi_query_cached_kv_attention(
377 - num_queries=11,
378 - num_heads=3,
379 - head_size=head_size,
380 - block_size=block_size,
381 - num_blocks=1024,
382 - dtype=dtype,
383 - )
384 -
295 # NOTE(woosuk): FlashAttention does not support FP32.
296 for dtype in [torch.half]:
297 # NOTE(woosuk): FlashAttention does not support head_size > 128.