@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0deacbce6e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Implement `single_query_cached_kv_attention` kernel (#3)

Woosuk Kwon committed Mar 1, 2023 at 15:02 UTC 0deacbce6e96a1af5885babc4e470ce2a0cecf95
12 files changed +2140 -60
cacheflow/master/block_manager.py
+3 -1
@@ -15,7 +15,9 @@ class BlockManager:
15 block_size: int,
16 num_blocks: int,
17 ) -> None:
18 - assert block_size in [8, 16, 32]
18 + if block_size not in [8, 16]:
19 + raise ValueError(f'Unsupported block size: {block_size}'
20 + 'The block size must be either 8 or 16.')
21 self.device = device
22 self.block_size = block_size
23 self.num_blocks = num_blocks
cacheflow/models/attention.py
+19 -35
@@ -3,7 +3,8 @@ from typing import List, Optional
3 import torch
4 import torch.nn as nn
5
6 -from cacheflow import ops
6 +from cacheflow import attention_ops
7 +from cacheflow import cache_ops
8 from cacheflow.models import InputMetadata
9
10
@@ -11,7 +12,7 @@ class OPTCacheFlowAttention(nn.Module):
12
13 def __init__(self, scale: float) -> None:
14 super().__init__()
14 - self.scale = scale
15 + self.scale = float(scale)
16
17 def _masked_attention(
18 self,
@@ -57,38 +58,21 @@ class OPTCacheFlowAttention(nn.Module):
58 output: torch.Tensor, # [num_generation_tokens, num_heads, head_size]
59 query: torch.Tensor, # [num_generation_tokens, num_heads, head_size]
60 key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
60 - value_cache: torch.Tensor, # [num_blocks, num_heads, block_size, head_size]
61 + value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
62 input_metadata: InputMetadata,
63 ) -> None:
63 - num_heads = value_cache.shape[1]
64 - head_size = value_cache.shape[3]
65 - block_size = value_cache.shape[2]
66 - block_tables = input_metadata.block_tables
67 -
68 - # FIXME(woosuk): Replace the following with a custom op.
69 - for i in range(input_metadata.num_generation_tokens):
70 - q = query[i].unsqueeze(0)
71 - block_table = block_tables[i]
72 - context_len = int(input_metadata.context_lens[i])
73 -
74 - keys = []
75 - values = []
76 - for j in range(context_len):
77 - block_number = int(block_table[j // block_size])
78 - block_offset = j % block_size
79 -
80 - k = key_cache[block_number, :, :, block_offset, :]
81 - k = k.reshape(num_heads, head_size)
82 - keys.append(k)
83 -
84 - v = value_cache[block_number, :, block_offset, :]
85 - values.append(v)
86 - keys = torch.stack(keys, dim=0)
87 - values = torch.stack(values, dim=0)
88 -
89 - out = self._masked_attention(q, keys, values)
90 - out = out.view(num_heads, head_size)
91 - output[i].copy_(out, non_blocking=True)
64 + block_size = value_cache.shape[3]
65 + attention_ops.single_query_cached_kv_attention(
66 + output,
67 + query,
68 + key_cache,
69 + value_cache,
70 + self.scale,
71 + input_metadata.block_tables,
72 + input_metadata.context_lens,
73 + block_size,
74 + input_metadata.max_context_len,
75 + )
76
77 def forward(
78 self,
@@ -96,7 +80,7 @@ class OPTCacheFlowAttention(nn.Module):
80 key: torch.Tensor, # [num_tokens, num_heads * head_size]
81 value: torch.Tensor, # [num_tokens, num_heads * head_size]
82 key_cache: torch.Tensor, # [num_blocks, num_heads, head_size/x, block_size, x]
99 - value_cache: torch.Tensor, # [num_blocks, num_heads, block_size, head_size]
83 + value_cache: torch.Tensor, # [num_blocks, num_heads, head_size, block_size]
84 input_metadata: InputMetadata,
85 cache_event: Optional[torch.cuda.Event],
86 ) -> torch.Tensor: # [num_tokens, num_heads * head_size]
@@ -110,7 +94,7 @@ class OPTCacheFlowAttention(nn.Module):
94
95 # Reshape the input tensors.
96 num_heads = value_cache.shape[1]
113 - head_size = value_cache.shape[3]
97 + head_size = value_cache.shape[2]
98 query = query.view(-1, num_heads, head_size)
99 key = key.view(-1, num_heads, head_size)
100 value = value.view(-1, num_heads, head_size)
@@ -125,7 +109,7 @@ class OPTCacheFlowAttention(nn.Module):
109 cache_event.wait()
110
111 # Reshape the keys and values and store them in the cache.
128 - ops.reshape_and_cache(
112 + cache_ops.reshape_and_cache(
113 key, value, key_cache, value_cache, input_metadata.slot_mapping)
114
115 if input_metadata.num_generation_tokens > 0:
cacheflow/worker/cache_engine.py
+12 -8
@@ -1,7 +1,7 @@
1 from typing import Dict, List, Tuple
2
3 import torch
4 -from cacheflow import ops
4 +from cacheflow import cache_ops
5
6 KVCache = Tuple[torch.Tensor, torch.Tensor]
7
@@ -57,20 +57,22 @@ class CacheEngine:
57 def get_value_block_shape(self) -> Tuple[int, int, int]:
58 return (
59 self.num_heads,
60 - self.block_size,
60 self.head_size,
61 + self.block_size,
62 )
63
64 def allocate_gpu_cache(self) -> List[KVCache]:
65 gpu_cache: List[KVCache] = []
66 + key_block_shape = self.get_key_block_shape()
67 + value_block_shape = self.get_value_block_shape()
68 for _ in range(self.num_layers):
69 key_blocks = torch.empty(
68 - size=(self.num_gpu_blocks, *self.get_key_block_shape()),
70 + size=(self.num_gpu_blocks, *key_block_shape),
71 dtype=self.dtype,
72 device=self.gpu_id,
73 )
74 value_blocks = torch.empty(
73 - size=(self.num_gpu_blocks, *self.get_value_block_shape()),
75 + size=(self.num_gpu_blocks, *value_block_shape),
76 dtype=self.dtype,
77 device=self.gpu_id,
78 )
@@ -79,14 +81,16 @@ class CacheEngine:
81
82 def allocate_cpu_cache(self) -> List[KVCache]:
83 cpu_cache: List[KVCache] = []
84 + key_block_shape = self.get_key_block_shape()
85 + value_block_shape = self.get_value_block_shape()
86 for _ in range(self.num_layers):
87 key_blocks = torch.empty(
84 - size=(self.num_cpu_blocks, *self.get_key_block_shape()),
88 + size=(self.num_cpu_blocks, *key_block_shape),
89 dtype=self.dtype,
90 pin_memory=True,
91 )
92 value_blocks = torch.empty(
89 - size=(self.num_cpu_blocks, *self.get_value_block_shape()),
93 + size=(self.num_cpu_blocks, *value_block_shape),
94 dtype=self.dtype,
95 pin_memory=True,
96 )
@@ -104,10 +108,10 @@ class CacheEngine:
108 src_key_cache, src_value_cache = src[i]
109 dst_key_cache, dst_value_cache = dst[i]
110 # Copy the key blocks.
107 - ops.copy_cache_blocks(
111 + cache_ops.copy_cache_blocks(
112 src_key_cache, dst_key_cache, src_to_dst)
113 # Copy the value blocks.
110 - ops.copy_cache_blocks(
114 + cache_ops.copy_cache_blocks(
115 src_value_cache, dst_value_cache, src_to_dst)
116 event = self.events[i]
117 event.record(stream=self.cache_stream)
cacheflow/worker/worker.py
+1 -1
@@ -118,7 +118,7 @@ class Worker:
118 _pad_to_max(block_table, max_num_blocks_per_seq)
119 for block_table in generation_block_tables]
120 block_tables_tensor = torch.tensor(
121 - padded_block_tables, dtype=int, device=self.device)
121 + padded_block_tables, dtype=torch.int, device=self.device)
122
123 input_metadata = InputMetadata(
124 seq_ids=prompt_seq_ids + generation_seq_ids,
csrc/attention.cpp new
+19
@@ -0,0 +1,19 @@
1 +#include <torch/extension.h>
2 +
3 +void single_query_cached_kv_attention(
4 + torch::Tensor& out,
5 + torch::Tensor& query,
6 + torch::Tensor& key_cache,
7 + torch::Tensor& value_cache,
8 + float scale,
9 + torch::Tensor& block_tables,
10 + torch::Tensor& context_lens,
11 + int block_size,
12 + int max_context_len);
13 +
14 +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
15 + m.def(
16 + "single_query_cached_kv_attention",
17 + &single_query_cached_kv_attention,
18 + "Compute the attention between an input query and the cached key/value tensors");
19 +}
csrc/attention_kernels.cu new
+400
@@ -0,0 +1,400 @@
1 +#include <torch/extension.h>
2 +#include <ATen/cuda/CUDAContext.h>
3 +
4 +#include "attention_utils.h"
5 +#include "cuda_primitives.h"
6 +
7 +#include <algorithm>
8 +
9 +#define WARP_SIZE 32
10 +
11 +namespace cacheflow {
12 +
13 +// Grid: (num_heads, num_seqs).
14 +template<
15 + typename scalar_t,
16 + int HEAD_SIZE,
17 + int BLOCK_SIZE,
18 + int NUM_THREADS>
19 +__global__ void single_query_cached_kv_attention_kernel(
20 + scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
21 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
22 + const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
23 + const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
24 + const float scale,
25 + const int* __restrict__ block_tables, // [num_seqs, max_num_blocks_per_seq]
26 + const int* __restrict__ context_lens, // [num_seqs]
27 + const int max_num_blocks_per_seq) {
28 + constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
29 + constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
30 + const int thread_idx = threadIdx.x;
31 + const int warp_idx = thread_idx / WARP_SIZE;
32 + const int lane = thread_idx % WARP_SIZE;
33 +
34 + const int head_idx = blockIdx.x;
35 + const int num_heads = gridDim.x;
36 + const int seq_idx = blockIdx.y;
37 +
38 + // A vector type to store a part of a key or a query.
39 + // The vector size is configured in such a way that the threads in a thread group
40 + // fetch or comput 16 bytes at a time.
41 + // For example, if the size of a thread group is 4 and the data type is half,
42 + // then the vector size is 16 / (4 * sizeof(half)) == 2.
43 + constexpr int VEC_SIZE = 16 / (THREAD_GROUP_SIZE * sizeof(scalar_t));
44 + using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
45 + using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
46 +
47 + constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
48 + constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
49 +
50 + const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
51 + const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
52 +
53 + // Load the query to registers.
54 + // Each thread in a thread group has a different part of the query.
55 + // For example, if the the thread group size is 4, then the first thread in the group
56 + // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
57 + // th vectors of the query, and so on.
58 + const scalar_t* q_ptr = q + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
59 + Q_vec q_vecs[NUM_VECS_PER_THREAD];
60 +#pragma unroll
61 + for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
62 + const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
63 + q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
64 + }
65 +
66 + // Memory planning.
67 + extern __shared__ char shared_mem[];
68 + // NOTE(woosuk): We use FP32 logits and accumulation.
69 + float *logits = reinterpret_cast<float*>(shared_mem);
70 + // Workspace for reduction.
71 + __shared__ float red_smem[2 * NUM_WARPS];
72 +
73 + // x == THREAD_GROUP_SIZE * VEC_SIZE
74 + // Each thread group fetches x elements from the key at a time.
75 + constexpr int x = 16 / sizeof(scalar_t);
76 + float qk_max = -FLT_MAX;
77 +
78 + const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
79 + const int context_len = context_lens[seq_idx];
80 + const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
81 +
82 + // Iterate over the key blocks.
83 + // Each warp fetches a block of keys for each iteration.
84 + // Each thread group in a warp fetches a key from the block, and computes
85 + // dot product with the query.
86 + for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
87 + const int physical_block_number = block_table[block_idx];
88 + const int physical_block_offset = thread_group_idx % BLOCK_SIZE;
89 + const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
90 +
91 + // Load a key to registers.
92 + // Each thread in a thread group has a different part of the key.
93 + // For example, if the the thread group size is 4, then the first thread in the group
94 + // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
95 + // vectors of the key, and so on.
96 + K_vec k_vecs[NUM_VECS_PER_THREAD];
97 +#pragma unroll
98 + for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
99 + const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
100 + + head_idx * HEAD_SIZE * BLOCK_SIZE
101 + + physical_block_offset * x;
102 + const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
103 + const int offset1 = (vec_idx * VEC_SIZE) / x;
104 + const int offset2 = (vec_idx * VEC_SIZE) % x;
105 + k_vecs[i] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
106 + }
107 +
108 + // Compute dot product.
109 + // This includes a reduction across the threads in the same thread group.
110 + const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
111 + const bool mask = token_idx >= context_len;
112 +
113 + if (thread_group_offset == 0) {
114 + // Store the partial reductions to shared memory.
115 + // NOTE(woosuk): It is required to zero out the masked logits.
116 + logits[token_idx] = mask ? 0.f : qk;
117 + // Update the max value.
118 + qk_max = mask ? qk_max : fmaxf(qk_max, qk);
119 + }
120 + }
121 +
122 + // Perform reduction across the threads in the same warp to get the
123 + // max qk value for each "warp" (not across the thread block yet).
124 + // The 0-th thread of each thread group already has its max qk value.
125 +#pragma unroll
126 + for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
127 + qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
128 + }
129 + if (lane == 0) {
130 + red_smem[warp_idx] = qk_max;
131 + }
132 + __syncthreads();
133 +
134 + // TODO(woosuk): Refactor this part.
135 + // Get the max qk value for the sequence.
136 + qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
137 +#pragma unroll
138 + for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
139 + qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
140 + }
141 + // Broadcast the max qk value to all threads.
142 + qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
143 +
144 + // Get the sum of the exp values.
145 + float exp_sum = 0.f;
146 + for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
147 + float val = __expf(logits[i] - qk_max);
148 + logits[i] = val;
149 + exp_sum += val;
150 + }
151 + exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
152 +
153 + // Compute softmax.
154 + const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
155 + for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
156 + logits[i] *= inv_sum;
157 + }
158 + __syncthreads();
159 +
160 + // Each thread will fetch 16 bytes from the value cache at a time.
161 + constexpr int V_VEC_SIZE = 16 / sizeof(scalar_t);
162 + using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
163 + using L_vec = typename FloatVec<V_vec>::Type;
164 +
165 + constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
166 + constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
167 + constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
168 +
169 + float accs[NUM_ROWS_PER_THREAD];
170 +#pragma unroll
171 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
172 + accs[i] = 0.f;
173 + }
174 +
175 + for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
176 + const int physical_block_number = block_table[block_idx];
177 + const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
178 + const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
179 + L_vec logits_vec = *reinterpret_cast<L_vec*>(logits + token_idx);
180 +
181 + const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
182 + + head_idx * HEAD_SIZE * BLOCK_SIZE;
183 +#pragma unroll
184 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
185 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
186 + if (row_idx < HEAD_SIZE) {
187 + const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
188 + V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
189 + accs[i] += dot(logits_vec, cast_to_float(v_vec));
190 + }
191 + }
192 + }
193 +
194 + // Perform reduction within each warp.
195 +#pragma unroll
196 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
197 + float acc = accs[i];
198 +#pragma unroll
199 + for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
200 + acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
201 + }
202 + accs[i] = acc;
203 + }
204 +
205 + // NOTE(woosuk): A barrier is required because the shared memory space for logits
206 + // is reused for the output.
207 + __syncthreads();
208 +
209 + // Perform reduction across warps.
210 + float* out_smem = reinterpret_cast<float*>(shared_mem);
211 +#pragma unroll
212 + for (int i = NUM_WARPS; i > 1; i /= 2) {
213 + int mid = i / 2;
214 + // Upper warps write to shared memory.
215 + if (warp_idx >= mid && warp_idx < i) {
216 + float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
217 +#pragma unroll
218 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
219 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
220 + if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
221 + dst[row_idx] = accs[i];
222 + }
223 + }
224 + }
225 + __syncthreads();
226 +
227 + // Lower warps update the output.
228 + if (warp_idx < mid) {
229 + const float* src = &out_smem[warp_idx * HEAD_SIZE];
230 +#pragma unroll
231 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
232 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
233 + if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
234 + accs[i] += src[row_idx];
235 + }
236 + }
237 + }
238 + __syncthreads();
239 + }
240 +
241 + // Write the final output.
242 + if (warp_idx == 0) {
243 + scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
244 +#pragma unroll
245 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
246 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
247 + if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
248 + convert_from_float(*(out_ptr + row_idx), accs[i]);
249 + }
250 + }
251 + }
252 +}
253 +
254 +} // namespace cacheflow
255 +
256 +#define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
257 + cacheflow::single_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
258 + <<<grid, block, shared_mem_size, stream>>>( \
259 + out_ptr, \
260 + query_ptr, \
261 + key_cache_ptr, \
262 + value_cache_ptr, \
263 + scale, \
264 + block_tables_ptr, \
265 + context_lens_ptr, \
266 + max_num_blocks_per_seq);
267 +
268 +// TODO(woosuk): Tune NUM_THREADS.
269 +template<
270 + typename T,
271 + int BLOCK_SIZE,
272 + int NUM_THREADS = 128>
273 +void single_query_cached_kv_attention_launcher(
274 + torch::Tensor& out,
275 + torch::Tensor& query,
276 + torch::Tensor& key_cache,
277 + torch::Tensor& value_cache,
278 + float scale,
279 + torch::Tensor& block_tables,
280 + torch::Tensor& context_lens,
281 + int max_context_len) {
282 + int num_seqs = query.size(0);
283 + int num_heads = query.size(1);
284 + int head_size = query.size(2);
285 + int max_num_blocks_per_seq = block_tables.size(1);
286 +
287 + T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
288 + T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
289 + T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
290 + T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
291 + int* block_tables_ptr = block_tables.data_ptr<int>();
292 + int* context_lens_ptr = context_lens.data_ptr<int>();
293 +
294 + constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
295 + int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
296 + int logits_size = padded_max_context_len * sizeof(float);
297 + int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
298 + int shared_mem_size = std::max(logits_size, outputs_size);
299 +
300 + dim3 grid(num_heads, num_seqs);
301 + dim3 block(NUM_THREADS);
302 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
303 + switch (head_size) {
304 + case 32:
305 + LAUNCH_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
306 + break;
307 + case 64:
308 + LAUNCH_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
309 + break;
310 + case 80:
311 + LAUNCH_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
312 + break;
313 + case 96:
314 + LAUNCH_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
315 + break;
316 + case 128:
317 + LAUNCH_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
318 + break;
319 + case 160:
320 + LAUNCH_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
321 + break;
322 + case 192:
323 + LAUNCH_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
324 + break;
325 + case 256:
326 + LAUNCH_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
327 + break;
328 + default:
329 + assert(false);
330 + break;
331 + }
332 +}
333 +
334 +void single_query_cached_kv_attention(
335 + torch::Tensor& out,
336 + torch::Tensor& query,
337 + torch::Tensor& key_cache,
338 + torch::Tensor& value_cache,
339 + float scale,
340 + torch::Tensor& block_tables,
341 + torch::Tensor& context_lens,
342 + int block_size,
343 + int max_context_len) {
344 + // TODO(woosuk): Support BF16.
345 + if (query.element_size() == 2) {
346 + // Half.
347 + if (block_size == 8) {
348 + single_query_cached_kv_attention_launcher<uint16_t, 8>(
349 + out,
350 + query,
351 + key_cache,
352 + value_cache,
353 + scale,
354 + block_tables,
355 + context_lens,
356 + max_context_len);
357 + } else if (block_size == 16) {
358 + single_query_cached_kv_attention_launcher<uint16_t, 16>(
359 + out,
360 + query,
361 + key_cache,
362 + value_cache,
363 + scale,
364 + block_tables,
365 + context_lens,
366 + max_context_len);
367 + } else {
368 + assert(false);
369 + }
370 + } else if (query.element_size() == 4) {
371 + // Float.
372 + if (block_size == 8) {
373 + single_query_cached_kv_attention_launcher<float, 8>(
374 + out,
375 + query,
376 + key_cache,
377 + value_cache,
378 + scale,
379 + block_tables,
380 + context_lens,
381 + max_context_len);
382 + } else if (block_size == 16) {
383 + single_query_cached_kv_attention_launcher<float, 16>(
384 + out,
385 + query,
386 + key_cache,
387 + value_cache,
388 + scale,
389 + block_tables,
390 + context_lens,
391 + max_context_len);
392 + } else {
393 + assert(false);
394 + }
395 + } else {
396 + assert(false);
397 + }
398 +}
399 +
400 +#undef WARP_SIZE
csrc/attention_utils.h new
+204
@@ -0,0 +1,204 @@
1 +#pragma once
2 +
3 +#include "cuda_primitives.h"
4 +
5 +#include <float.h>
6 +#include <type_traits>
7 +
8 +#define MMHA_USE_FP32_ACUM_FOR_FMA
9 +#define MMHA_USE_FP32_ACUM_FOR_OUT
10 +
11 +namespace cacheflow {
12 +
13 +// A vector type to store Q, K, V elements.
14 +template<typename T, int VEC_SIZE>
15 +struct Vec {};
16 +template<>
17 +struct Vec<float, 1> {
18 + using Type = float;
19 +};
20 +template<>
21 +struct Vec<float, 2> {
22 + using Type = float2;
23 +};
24 +template<>
25 +struct Vec<float, 4> {
26 + using Type = float4;
27 +};
28 +template<>
29 +struct Vec<uint16_t, 1> {
30 + using Type = uint16_t;
31 +};
32 +template<>
33 +struct Vec<uint16_t, 2> {
34 + using Type = uint32_t;
35 +};
36 +template<>
37 +struct Vec<uint16_t, 4> {
38 + using Type = uint2;
39 +};
40 +template<>
41 +struct Vec<uint16_t, 8> {
42 + using Type = uint4;
43 +};
44 +
45 +template<typename T>
46 +struct FloatVec {};
47 +template<>
48 +struct FloatVec<float> {
49 + using Type = float;
50 +};
51 +template<>
52 +struct FloatVec<float2> {
53 + using Type = float2;
54 +};
55 +template<>
56 +struct FloatVec<float4> {
57 + using Type = float4;
58 +};
59 +template<>
60 +struct FloatVec<uint16_t> {
61 + using Type = float;
62 +};
63 +template<>
64 +struct FloatVec<uint32_t> {
65 + using Type = float2;
66 +};
67 +template<>
68 +struct FloatVec<uint2> {
69 + using Type = Float4_;
70 +};
71 +template<>
72 +struct FloatVec<uint4> {
73 + using Type = Float8_;
74 +};
75 +
76 +template<int THREADS_PER_KEY, typename K_vec, int N>
77 +inline __device__ float qk_dot_(const K_vec (&q)[N], const K_vec (&k)[N])
78 +{
79 + using K_vec_acum = typename FloatVec<K_vec>::Type;
80 + // Compute the parallel products for Q*K^T (treat vector lanes separately).
81 + K_vec_acum qk_vec = mul<K_vec_acum, K_vec, K_vec>(q[0], k[0]);
82 +#pragma unroll
83 + for (int ii = 1; ii < N; ++ii) {
84 + qk_vec = fma(q[ii], k[ii], qk_vec);
85 + }
86 +
87 + // Finalize the reduction across lanes.
88 + float qk = sum(qk_vec);
89 +#pragma unroll
90 + for (int mask = THREADS_PER_KEY / 2; mask >= 1; mask /= 2) {
91 + qk += __shfl_xor_sync(uint32_t(-1), qk, mask);
92 + }
93 + return qk;
94 +}
95 +
96 +////////////////////////////////////////////////////////////////////////////////////////////////////
97 +
98 +template<typename T, int THREADS_PER_KEY>
99 +struct Qk_dot {
100 + template<typename K_vec, int N>
101 + static inline __device__ float dot(const K_vec (&q)[N], const K_vec (&k)[N])
102 + {
103 + return qk_dot_<THREADS_PER_KEY>(q, k);
104 + }
105 +};
106 +
107 +////////////////////////////////////////////////////////////////////////////////////////////////////
108 +
109 +inline __device__ float4 hmma_fp32(const uint2& a, uint32_t b)
110 +{
111 + float4 c;
112 + float zero = 0.f;
113 + asm volatile("mma.sync.aligned.m16n8k8.row.col.f32.f16.f16.f32 \n"
114 + " {%0, %1, %2, %3}, \n"
115 + " {%4, %5}, \n"
116 + " {%6}, \n"
117 + " {%7, %7, %7, %7}; \n"
118 +
119 + : "=f"(c.x), "=f"(c.y), "=f"(c.z), "=f"(c.w)
120 + : "r"(a.x) "r"(a.y), "r"(b), "f"(zero));
121 + return c;
122 +}
123 +
124 +////////////////////////////////////////////////////////////////////////////////////////////////////
125 +
126 +template<int N>
127 +inline __device__ float qk_hmma_dot_(const uint32_t (&q)[N], const uint32_t (&k)[N])
128 +{
129 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 750
130 + using K_vec_acum = typename FloatVec<uint32_t>::Type;
131 + K_vec_acum qk_vec = mul<K_vec_acum, uint32_t, uint32_t>(q[0], k[0]);
132 +#pragma unroll
133 + for (int ii = 1; ii < N; ++ii) {
134 + qk_vec = fma(q[ii], k[ii], qk_vec);
135 + }
136 +#ifdef MMHA_USE_FP32_ACUM_FOR_FMA
137 + uint32_t qk_vec_ = float2_to_half2(qk_vec);
138 + return hmma_fp32(make_uint2(qk_vec_, 0u), 0x3c003c00u).x;
139 +#else
140 + return hmma_fp32(make_uint2(qk_vec, 0u), 0x3c003c00u).x;
141 +#endif
142 +#else
143 + return 0.f;
144 +#endif
145 +}
146 +
147 +////////////////////////////////////////////////////////////////////////////////////////////////////
148 +
149 +template<>
150 +struct Qk_dot<uint16_t, 4> {
151 + template<int N>
152 + static inline __device__ float dot(const uint32_t (&q)[N], const uint32_t (&k)[N])
153 + {
154 +#if __CUDA_ARCH__ >= 750 && defined(MMHA_USE_HMMA_FOR_REDUCTION)
155 + return qk_hmma_dot_(q, k);
156 +#else
157 + return qk_dot_<4>(q, k);
158 +#endif // defined MMHA_USE_HMMA_FOR_REDUCTION
159 + }
160 +};
161 +
162 +////////////////////////////////////////////////////////////////////////////////////////////////////
163 +
164 +template<int WARPS_PER_BLOCK, int WARP_SIZE = 32>
165 +inline __device__ float block_sum(float* red_smem, float sum)
166 +{
167 +
168 + // Decompose the thread index into warp / lane.
169 + int warp = threadIdx.x / WARP_SIZE;
170 + int lane = threadIdx.x % WARP_SIZE;
171 +
172 + // Compute the sum per warp.
173 +#pragma unroll
174 + for (int mask = WARP_SIZE / 2; mask >= 1; mask /= 2) {
175 + sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
176 + }
177 +
178 + // Warp leaders store the data to shared memory.
179 + if (lane == 0) {
180 + red_smem[warp] = sum;
181 + }
182 +
183 + // Make sure the data is in shared memory.
184 + __syncthreads();
185 +
186 + // The warps compute the final sums.
187 + if (lane < WARPS_PER_BLOCK) {
188 + sum = red_smem[lane];
189 + }
190 +
191 +// Parallel reduction inside the warp.
192 +#pragma unroll
193 + for (int mask = WARPS_PER_BLOCK / 2; mask >= 1; mask /= 2) {
194 + sum += __shfl_xor_sync(uint32_t(-1), sum, mask);
195 + }
196 +
197 + // Broadcast to other threads.
198 + return __shfl_sync(uint32_t(-1), sum, 0);
199 +}
200 +
201 +} // namespace cacheflow
202 +
203 +#undef MMHA_USE_FP32_ACUM_FOR_FMA
204 +#undef MMHA_USE_FP32_ACUM_FOR_OUT
csrc/cache_kernels.cu
+5 -5
@@ -48,7 +48,7 @@ __global__ void reshape_and_cache_kernel(
48 const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
49 const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
50 scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
51 - scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, block_size, head_size]
51 + scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
52 const int* __restrict__ slot_mapping, // [num_tokens]
53 const int num_heads,
54 const int head_size,
@@ -73,10 +73,10 @@ __global__ void reshape_and_cache_kernel(
73 + x_idx * block_size * x
74 + block_offset * x
75 + x_offset;
76 - const int tgt_value_idx = block_idx * num_heads * block_size * head_size
77 - + head_idx * block_size * head_size
78 - + block_offset * head_size
79 - + head_offset;
76 + const int tgt_value_idx = block_idx * num_heads * head_size * block_size
77 + + head_idx * head_size * block_size
78 + + head_offset * block_size
79 + + block_offset;
80 key_cache[tgt_key_idx] = __ldg(&key[src_idx]);
81 value_cache[tgt_value_idx] = __ldg(&value[src_idx]);
82 }
csrc/cuda_primitives.h new
+1318
@@ -0,0 +1,1318 @@
1 +#pragma once
2 +
3 +#include <stdint.h>
4 +
5 +namespace cacheflow {
6 +////////////////////////////////////////////////////////////////////////////////////////////////////
7 +
8 +struct Float8_ {
9 + float2 x;
10 + float2 y;
11 + float2 z;
12 + float2 w;
13 +};
14 +
15 +////////////////////////////////////////////////////////////////////////////////////////////////////
16 +
17 +struct Float4_ {
18 + float2 x;
19 + float2 y;
20 +};
21 +
22 +////////////////////////////////////////////////////////////////////////////////////////////////////
23 +
24 +#ifdef ENABLE_BF16
25 +struct bf16_4_t {
26 + __nv_bfloat162 x;
27 + __nv_bfloat162 y;
28 +};
29 +
30 +////////////////////////////////////////////////////////////////////////////////////////////////////
31 +
32 +struct bf16_8_t {
33 + __nv_bfloat162 x;
34 + __nv_bfloat162 y;
35 + __nv_bfloat162 z;
36 + __nv_bfloat162 w;
37 +};
38 +#endif
39 +
40 +////////////////////////////////////////////////////////////////////////////////////////////////////
41 +
42 +inline __device__ float add(float a, float b)
43 +{
44 + return a + b;
45 +}
46 +
47 +////////////////////////////////////////////////////////////////////////////////////////////////////
48 +
49 +inline __device__ float2 add(float2 a, float2 b)
50 +{
51 + float2 c;
52 + c.x = add(a.x, b.x);
53 + c.y = add(a.y, b.y);
54 + return c;
55 +}
56 +
57 +////////////////////////////////////////////////////////////////////////////////////////////////////
58 +
59 +inline __device__ float4 add(float4 a, float4 b)
60 +{
61 + float4 c;
62 + c.x = add(a.x, b.x);
63 + c.y = add(a.y, b.y);
64 + c.z = add(a.z, b.z);
65 + c.w = add(a.w, b.w);
66 + return c;
67 +}
68 +
69 +////////////////////////////////////////////////////////////////////////////////////////////////////
70 +
71 +#ifdef ENABLE_BF16
72 +inline __device__ __nv_bfloat16 add(__nv_bfloat16 a, __nv_bfloat16 b)
73 +{
74 + return a + b;
75 +}
76 +
77 +////////////////////////////////////////////////////////////////////////////////////////////////////
78 +
79 +inline __device__ __nv_bfloat162 add(__nv_bfloat162 a, __nv_bfloat162 b)
80 +{
81 + return bf16hadd2(a, b);
82 +}
83 +
84 +////////////////////////////////////////////////////////////////////////////////////////////////////
85 +
86 +inline __device__ bf16_4_t add(bf16_4_t a, bf16_4_t b)
87 +{
88 + bf16_4_t c;
89 + c.x = add(a.x, b.x);
90 + c.y = add(a.y, b.y);
91 + return c;
92 +}
93 +
94 +////////////////////////////////////////////////////////////////////////////////////////////////////
95 +
96 +inline __device__ bf16_8_t add(bf16_8_t a, bf16_8_t b)
97 +{
98 + bf16_8_t c;
99 + c.x = add(a.x, b.x);
100 + c.y = add(a.y, b.y);
101 + c.z = add(a.z, b.z);
102 + c.w = add(a.w, b.w);
103 + return c;
104 +}
105 +#endif // ENABLE_BF16
106 +
107 +////////////////////////////////////////////////////////////////////////////////////////////////////
108 +
109 +inline __device__ uint16_t add(uint16_t a, uint16_t b)
110 +{
111 + uint16_t c;
112 + asm volatile("add.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
113 + return c;
114 +}
115 +
116 +////////////////////////////////////////////////////////////////////////////////////////////////////
117 +
118 +inline __device__ uint32_t add(uint32_t a, uint32_t b)
119 +{
120 + uint32_t c;
121 + asm volatile("add.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
122 + return c;
123 +}
124 +
125 +////////////////////////////////////////////////////////////////////////////////////////////////////
126 +
127 +inline __device__ uint2 add(uint2 a, uint2 b)
128 +{
129 + uint2 c;
130 + c.x = add(a.x, b.x);
131 + c.y = add(a.y, b.y);
132 + return c;
133 +}
134 +
135 +////////////////////////////////////////////////////////////////////////////////////////////////////
136 +
137 +inline __device__ uint4 add(uint4 a, uint4 b)
138 +{
139 + uint4 c;
140 + c.x = add(a.x, b.x);
141 + c.y = add(a.y, b.y);
142 + c.z = add(a.z, b.z);
143 + c.w = add(a.w, b.w);
144 + return c;
145 +}
146 +
147 +////////////////////////////////////////////////////////////////////////////////////////////////////
148 +
149 +inline __device__ uint16_t float_to_half(float f)
150 +{
151 + union {
152 + uint32_t u32;
153 + uint16_t u16[2];
154 + } tmp;
155 +#if 0 && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800 // Is it better?
156 + float zero = 0.f;
157 + asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(zero), "f"(f));
158 +#else
159 + asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f));
160 +#endif
161 + return tmp.u16[0];
162 +}
163 +
164 +////////////////////////////////////////////////////////////////////////////////////////////////////
165 +
166 +inline __device__ uint32_t float2_to_half2(float2 f)
167 +{
168 + union {
169 + uint32_t u32;
170 + uint16_t u16[2];
171 + } tmp;
172 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
173 + asm volatile("cvt.rn.f16x2.f32 %0, %1, %2;\n" : "=r"(tmp.u32) : "f"(f.y), "f"(f.x));
174 +#else
175 + asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[0]) : "f"(f.x));
176 + asm volatile("cvt.rn.f16.f32 %0, %1;\n" : "=h"(tmp.u16[1]) : "f"(f.y));
177 +#endif
178 + return tmp.u32;
179 +}
180 +
181 +////////////////////////////////////////////////////////////////////////////////////////////////////
182 +
183 +inline __device__ float half_to_float(uint16_t h)
184 +{
185 + float f;
186 + asm volatile("cvt.f32.f16 %0, %1;\n" : "=f"(f) : "h"(h));
187 + return f;
188 +}
189 +
190 +////////////////////////////////////////////////////////////////////////////////////////////////////
191 +
192 +inline __device__ float2 half2_to_float2(uint32_t v)
193 +{
194 + uint16_t lo, hi;
195 + asm volatile("mov.b32 {%0, %1}, %2;\n" : "=h"(lo), "=h"(hi) : "r"(v));
196 + return make_float2(half_to_float(lo), half_to_float(hi));
197 +}
198 +
199 +////////////////////////////////////////////////////////////////////////////////////////////////////
200 +
201 +inline __device__ float2 add(uint32_t a, float2 fb)
202 +{
203 + float2 fa = half2_to_float2(a);
204 + return add(fa, fb);
205 +}
206 +
207 +////////////////////////////////////////////////////////////////////////////////////////////////////
208 +
209 +inline __device__ Float4_ add(uint2 a, Float4_ fb)
210 +{
211 + Float4_ fc;
212 + fc.x = add(a.x, fb.x);
213 + fc.y = add(a.y, fb.y);
214 + return fc;
215 +}
216 +
217 +////////////////////////////////////////////////////////////////////////////////////////////////////
218 +
219 +inline __device__ Float8_ add(uint4 a, Float8_ fb)
220 +{
221 + Float8_ fc;
222 + fc.x = add(a.x, fb.x);
223 + fc.y = add(a.y, fb.y);
224 + fc.z = add(a.z, fb.z);
225 + fc.w = add(a.w, fb.w);
226 + return fc;
227 +}
228 +
229 +////////////////////////////////////////////////////////////////////////////////////////////////////
230 +
231 +inline __device__ uint32_t h0_h0(uint16_t a)
232 +{
233 + uint32_t b;
234 + asm volatile("mov.b32 %0, {%1, %1};" : "=r"(b) : "h"(a));
235 + return b;
236 +}
237 +
238 +////////////////////////////////////////////////////////////////////////////////////////////////////
239 +
240 +inline __device__ float fma(float a, float b, float c)
241 +{
242 + return a * b + c;
243 +}
244 +
245 +////////////////////////////////////////////////////////////////////////////////////////////////////
246 +
247 +inline __device__ float2 fma(float2 a, float2 b, float2 c)
248 +{
249 + float2 d;
250 + d.x = fma(a.x, b.x, c.x);
251 + d.y = fma(a.y, b.y, c.y);
252 + return d;
253 +}
254 +
255 +////////////////////////////////////////////////////////////////////////////////////////////////////
256 +
257 +inline __device__ float2 fma(float a, float2 b, float2 c)
258 +{
259 + float2 d;
260 + d.x = fma(a, b.x, c.x);
261 + d.y = fma(a, b.y, c.y);
262 + return d;
263 +}
264 +
265 +////////////////////////////////////////////////////////////////////////////////////////////////////
266 +
267 +inline __device__ float4 fma(float4 a, float4 b, float4 c)
268 +{
269 + float4 d;
270 + d.x = fma(a.x, b.x, c.x);
271 + d.y = fma(a.y, b.y, c.y);
272 + d.z = fma(a.z, b.z, c.z);
273 + d.w = fma(a.w, b.w, c.w);
274 + return d;
275 +}
276 +
277 +////////////////////////////////////////////////////////////////////////////////////////////////////
278 +
279 +inline __device__ float4 fma(float a, float4 b, float4 c)
280 +{
281 + float4 d;
282 + d.x = fma(a, b.x, c.x);
283 + d.y = fma(a, b.y, c.y);
284 + d.z = fma(a, b.z, c.z);
285 + d.w = fma(a, b.w, c.w);
286 + return d;
287 +}
288 +
289 +////////////////////////////////////////////////////////////////////////////////////////////////////
290 +
291 +inline __device__ Float4_ fma(float a, Float4_ b, Float4_ c)
292 +{
293 + Float4_ d;
294 + d.x = fma(a, b.x, c.x);
295 + d.y = fma(a, b.y, c.y);
296 + return d;
297 +}
298 +
299 +////////////////////////////////////////////////////////////////////////////////////////////////////
300 +
301 +inline __device__ Float8_ fma(float a, Float8_ b, Float8_ c)
302 +{
303 + Float8_ d;
304 + d.x = fma(a, b.x, c.x);
305 + d.y = fma(a, b.y, c.y);
306 + d.z = fma(a, b.z, c.z);
307 + d.w = fma(a, b.w, c.w);
308 + return d;
309 +}
310 +
311 +////////////////////////////////////////////////////////////////////////////////////////////////////
312 +
313 +#ifdef ENABLE_BF16
314 +inline __device__ float2 add(__nv_bfloat162 a, float2 fb)
315 +{
316 + float2 fa = bf1622float2(a);
317 + return add(fa, fb);
318 +}
319 +
320 +////////////////////////////////////////////////////////////////////////////////////////////////////
321 +
322 +inline __device__ Float4_ add(bf16_4_t a, Float4_ fb)
323 +{
324 + Float4_ fc;
325 + fc.x = add(a.x, fb.x);
326 + fc.y = add(a.y, fb.y);
327 + return fc;
328 +}
329 +
330 +////////////////////////////////////////////////////////////////////////////////////////////////////
331 +
332 +inline __device__ Float8_ add(bf16_8_t a, Float8_ fb)
333 +{
334 + Float8_ fc;
335 + fc.x = add(a.x, fb.x);
336 + fc.y = add(a.y, fb.y);
337 + fc.z = add(a.z, fb.z);
338 + fc.w = add(a.w, fb.w);
339 + return fc;
340 +}
341 +#endif // ENABLE_BF16
342 +
343 +////////////////////////////////////////////////////////////////////////////////////////////////////
344 +
345 +inline __device__ uint32_t fma(uint32_t a, uint32_t b, uint32_t c)
346 +{
347 + uint32_t d;
348 + asm volatile("fma.rn.f16x2 %0, %1, %2, %3;\n" : "=r"(d) : "r"(a), "r"(b), "r"(c));
349 + return d;
350 +}
351 +
352 +////////////////////////////////////////////////////////////////////////////////////////////////////
353 +
354 +inline __device__ uint32_t fma(uint16_t a, uint32_t b, uint32_t c)
355 +{
356 + return fma(h0_h0(a), b, c);
357 +}
358 +
359 +////////////////////////////////////////////////////////////////////////////////////////////////////
360 +
361 +inline __device__ uint2 fma(uint2 a, uint2 b, uint2 c)
362 +{
363 + uint2 d;
364 + d.x = fma(a.x, b.x, c.x);
365 + d.y = fma(a.y, b.y, c.y);
366 + return d;
367 +}
368 +
369 +////////////////////////////////////////////////////////////////////////////////////////////////////
370 +
371 +inline __device__ uint2 fma(uint16_t a, uint2 b, uint2 c)
372 +{
373 + uint32_t s = h0_h0(a);
374 + uint2 d;
375 + d.x = fma(s, b.x, c.x);
376 + d.y = fma(s, b.y, c.y);
377 + return d;
378 +}
379 +
380 +////////////////////////////////////////////////////////////////////////////////////////////////////
381 +
382 +inline __device__ uint4 fma(uint4 a, uint4 b, uint4 c)
383 +{
384 + uint4 d;
385 + d.x = fma(a.x, b.x, c.x);
386 + d.y = fma(a.y, b.y, c.y);
387 + d.z = fma(a.z, b.z, c.z);
388 + d.w = fma(a.w, b.w, c.w);
389 + return d;
390 +}
391 +
392 +////////////////////////////////////////////////////////////////////////////////////////////////////
393 +
394 +inline __device__ uint4 fma(uint16_t a, uint4 b, uint4 c)
395 +{
396 + uint32_t s = h0_h0(a);
397 + uint4 d;
398 + d.x = fma(s, b.x, c.x);
399 + d.y = fma(s, b.y, c.y);
400 + d.z = fma(s, b.z, c.z);
401 + d.w = fma(s, b.w, c.w);
402 + return d;
403 +}
404 +
405 +////////////////////////////////////////////////////////////////////////////////////////////////////
406 +
407 +inline __device__ float fma(uint16_t a, uint16_t b, float fc)
408 +{
409 + float fa = half_to_float(a);
410 + float fb = half_to_float(b);
411 + return fa * fb + fc;
412 +}
413 +
414 +////////////////////////////////////////////////////////////////////////////////////////////////////
415 +
416 +inline __device__ float2 fma(uint32_t a, uint32_t b, float2 fc)
417 +{
418 + float2 fa = half2_to_float2(a);
419 + float2 fb = half2_to_float2(b);
420 + return fma(fa, fb, fc);
421 +}
422 +
423 +////////////////////////////////////////////////////////////////////////////////////////////////////
424 +
425 +inline __device__ float2 fma(uint16_t a, uint32_t b, float2 fc)
426 +{
427 + return fma(h0_h0(a), b, fc);
428 +}
429 +
430 +////////////////////////////////////////////////////////////////////////////////////////////////////
431 +
432 +inline __device__ Float4_ fma(uint2 a, uint2 b, Float4_ fc)
433 +{
434 + Float4_ fd;
435 + fd.x = fma(a.x, b.x, fc.x);
436 + fd.y = fma(a.y, b.y, fc.y);
437 + return fd;
438 +}
439 +
440 +////////////////////////////////////////////////////////////////////////////////////////////////////
441 +
442 +inline __device__ Float4_ fma(uint16_t a, uint2 b, Float4_ fc)
443 +{
444 + uint32_t s = h0_h0(a);
445 + Float4_ fd;
446 + fd.x = fma(s, b.x, fc.x);
447 + fd.y = fma(s, b.y, fc.y);
448 + return fd;
449 +}
450 +
451 +////////////////////////////////////////////////////////////////////////////////////////////////////
452 +
453 +inline __device__ Float8_ fma(uint4 a, uint4 b, Float8_ fc)
454 +{
455 + Float8_ fd;
456 + fd.x = fma(a.x, b.x, fc.x);
457 + fd.y = fma(a.y, b.y, fc.y);
458 + fd.z = fma(a.z, b.z, fc.z);
459 + fd.w = fma(a.w, b.w, fc.w);
460 + return fd;
461 +}
462 +
463 +////////////////////////////////////////////////////////////////////////////////////////////////////
464 +
465 +inline __device__ Float8_ fma(uint16_t a, uint4 b, Float8_ fc)
466 +{
467 + uint32_t s = h0_h0(a);
468 + Float8_ fd;
469 + fd.x = fma(s, b.x, fc.x);
470 + fd.y = fma(s, b.y, fc.y);
471 + fd.z = fma(s, b.z, fc.z);
472 + fd.w = fma(s, b.w, fc.w);
473 + return fd;
474 +}
475 +
476 +////////////////////////////////////////////////////////////////////////////////////////////////////
477 +#ifdef ENABLE_BF16
478 +inline __device__ __nv_bfloat162 fma(__nv_bfloat162 a, __nv_bfloat162 b, __nv_bfloat162 c)
479 +{
480 + return bf16hfma2(a, b, c);
481 +}
482 +
483 +////////////////////////////////////////////////////////////////////////////////////////////////////
484 +
485 +inline __device__ __nv_bfloat162 fma(__nv_bfloat16 a, __nv_bfloat162 b, __nv_bfloat162 c)
486 +{
487 + return bf16hfma2(bf162bf162(a), b, c);
488 +}
489 +////////////////////////////////////////////////////////////////////////////////////////////////////
490 +
491 +inline __device__ bf16_4_t fma(bf16_4_t a, bf16_4_t b, bf16_4_t c)
492 +{
493 + bf16_4_t d;
494 + d.x = fma(a.x, b.x, c.x);
495 + d.y = fma(a.y, b.y, c.y);
496 + return d;
497 +}
498 +
499 +////////////////////////////////////////////////////////////////////////////////////////////////////
500 +
501 +inline __device__ bf16_4_t fma(__nv_bfloat16 a, bf16_4_t b, bf16_4_t c)
502 +{
503 + __nv_bfloat162 s = bf162bf162(a);
504 + bf16_4_t d;
505 + d.x = fma(s, b.x, c.x);
506 + d.y = fma(s, b.y, c.y);
507 + return d;
508 +}
509 +
510 +////////////////////////////////////////////////////////////////////////////////////////////////////
511 +
512 +inline __device__ bf16_8_t fma(bf16_8_t a, bf16_8_t b, bf16_8_t c)
513 +{
514 + bf16_8_t d;
515 + d.x = fma(a.x, b.x, c.x);
516 + d.y = fma(a.y, b.y, c.y);
517 + d.z = fma(a.z, b.z, c.z);
518 + d.w = fma(a.w, b.w, c.w);
519 + return d;
520 +}
521 +
522 +////////////////////////////////////////////////////////////////////////////////////////////////////
523 +
524 +inline __device__ bf16_8_t fma(__nv_bfloat16 a, bf16_8_t b, bf16_8_t c)
525 +{
526 + __nv_bfloat162 s = bf162bf162(a);
527 + bf16_8_t d;
528 + d.x = fma(s, b.x, c.x);
529 + d.y = fma(s, b.y, c.y);
530 + d.z = fma(s, b.z, c.z);
531 + d.w = fma(s, b.w, c.w);
532 + return d;
533 +}
534 +
535 +////////////////////////////////////////////////////////////////////////////////////////////////////
536 +
537 +inline __device__ float fma(__nv_bfloat16 a, __nv_bfloat16 b, float fc)
538 +{
539 + return __bfloat162float(a) * __bfloat162float(b) + fc;
540 +}
541 +
542 +////////////////////////////////////////////////////////////////////////////////////////////////////
543 +
544 +inline __device__ float2 fma(__nv_bfloat162 a, __nv_bfloat162 b, float2 fc)
545 +{
546 + float2 fa = bf1622float2(a);
547 + float2 fb = bf1622float2(b);
548 + return fma(fa, fb, fc);
549 +}
550 +
551 +////////////////////////////////////////////////////////////////////////////////////////////////////
552 +
553 +inline __device__ float2 fma(__nv_bfloat16 a, __nv_bfloat162 b, float2 fc)
554 +{
555 + return fma(bf162bf162(a), b, fc);
556 +}
557 +
558 +////////////////////////////////////////////////////////////////////////////////////////////////////
559 +
560 +inline __device__ Float4_ fma(bf16_4_t a, bf16_4_t b, Float4_ fc)
561 +{
562 + Float4_ fd;
563 + fd.x = fma(a.x, b.x, fc.x);
564 + fd.y = fma(a.y, b.y, fc.y);
565 + return fd;
566 +}
567 +
568 +////////////////////////////////////////////////////////////////////////////////////////////////////
569 +
570 +inline __device__ Float4_ fma(__nv_bfloat16 a, bf16_4_t b, Float4_ fc)
571 +{
572 + __nv_bfloat162 s = bf162bf162(a);
573 + Float4_ fd;
574 + fd.x = fma(s, b.x, fc.x);
575 + fd.y = fma(s, b.y, fc.y);
576 + return fd;
577 +}
578 +
579 +////////////////////////////////////////////////////////////////////////////////////////////////////
580 +
581 +inline __device__ Float8_ fma(bf16_8_t a, bf16_8_t b, Float8_ fc)
582 +{
583 + Float8_ fd;
584 + fd.x = fma(a.x, b.x, fc.x);
585 + fd.y = fma(a.y, b.y, fc.y);
586 + fd.z = fma(a.z, b.z, fc.z);
587 + fd.w = fma(a.w, b.w, fc.w);
588 + return fd;
589 +}
590 +
591 +////////////////////////////////////////////////////////////////////////////////////////////////////
592 +
593 +inline __device__ Float8_ fma(__nv_bfloat16 a, bf16_8_t b, Float8_ fc)
594 +{
595 + __nv_bfloat162 s = bf162bf162(a);
596 + Float8_ fd;
597 + fd.x = fma(s, b.x, fc.x);
598 + fd.y = fma(s, b.y, fc.y);
599 + fd.z = fma(s, b.z, fc.z);
600 + fd.w = fma(s, b.w, fc.w);
601 + return fd;
602 +}
603 +#endif // ENABLE_BF16
604 +////////////////////////////////////////////////////////////////////////////////////////////////////
605 +
606 +template<typename Acc, typename A, typename B>
607 +inline __device__ Acc mul(A a, B b);
608 +
609 +////////////////////////////////////////////////////////////////////////////////////////////////////
610 +
611 +template<>
612 +inline __device__ float mul<float, float>(float a, float b)
613 +{
614 + return a * b;
615 +}
616 +
617 +////////////////////////////////////////////////////////////////////////////////////////////////////
618 +
619 +template<>
620 +inline __device__ float2 mul(float2 a, float2 b)
621 +{
622 + float2 c;
623 + c.x = a.x * b.x;
624 + c.y = a.y * b.y;
625 + return c;
626 +}
627 +
628 +////////////////////////////////////////////////////////////////////////////////////////////////////
629 +
630 +template<>
631 +inline __device__ float2 mul(float a, float2 b)
632 +{
633 + float2 c;
634 + c.x = a * b.x;
635 + c.y = a * b.y;
636 + return c;
637 +}
638 +
639 +////////////////////////////////////////////////////////////////////////////////////////////////////
640 +
641 +template<>
642 +inline __device__ float4 mul(float4 a, float4 b)
643 +{
644 + float4 c;
645 + c.x = a.x * b.x;
646 + c.y = a.y * b.y;
647 + c.z = a.z * b.z;
648 + c.w = a.w * b.w;
649 + return c;
650 +}
651 +
652 +////////////////////////////////////////////////////////////////////////////////////////////////////
653 +
654 +template<>
655 +inline __device__ float4 mul(float a, float4 b)
656 +{
657 + float4 c;
658 + c.x = a * b.x;
659 + c.y = a * b.y;
660 + c.z = a * b.z;
661 + c.w = a * b.w;
662 + return c;
663 +}
664 +
665 +////////////////////////////////////////////////////////////////////////////////////////////////////
666 +
667 +template<>
668 +inline __device__ uint16_t mul(uint16_t a, uint16_t b)
669 +{
670 + uint16_t c;
671 + asm volatile("mul.f16 %0, %1, %2;\n" : "=h"(c) : "h"(a), "h"(b));
672 + return c;
673 +}
674 +
675 +////////////////////////////////////////////////////////////////////////////////////////////////////
676 +
677 +template<>
678 +inline __device__ uint32_t mul(uint32_t a, uint32_t b)
679 +{
680 + uint32_t c;
681 + asm volatile("mul.f16x2 %0, %1, %2;\n" : "=r"(c) : "r"(a), "r"(b));
682 + return c;
683 +}
684 +
685 +////////////////////////////////////////////////////////////////////////////////////////////////////
686 +
687 +template<>
688 +inline __device__ uint32_t mul(uint16_t a, uint32_t b)
689 +{
690 + return mul<uint32_t, uint32_t, uint32_t>(h0_h0(a), b);
691 +}
692 +
693 +////////////////////////////////////////////////////////////////////////////////////////////////////
694 +
695 +template<>
696 +inline __device__ uint2 mul(uint2 a, uint2 b)
697 +{
698 + uint2 c;
699 + c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
700 + c.y = mul<uint32_t, uint32_t, uint32_t>(a.y, b.y);
701 + return c;
702 +}
703 +
704 +////////////////////////////////////////////////////////////////////////////////////////////////////
705 +
706 +template<>
707 +inline __device__ uint2 mul(uint16_t a, uint2 b)
708 +{
709 + uint32_t s = h0_h0(a);
710 + uint2 c;
711 + c.x = mul<uint32_t, uint32_t, uint32_t>(s, b.x);
712 + c.y = mul<uint32_t, uint32_t, uint32_t>(s, b.y);
713 + return c;
714 +}
715 +
716 +////////////////////////////////////////////////////////////////////////////////////////////////////
717 +
718 +template<>
719 +inline __device__ uint4 mul(uint4 a, uint4 b)
720 +{
721 + uint4 c;
722 + c.x = mul<uint32_t, uint32_t, uint32_t>(a.x, b.x);
723 + c.y = mul<uint32_t, uint32_t, uint32_t>(a.y, b.y);
724 + c.z = mul<uint32_t, uint32_t, uint32_t>(a.z, b.z);
725 + c.w = mul<uint32_t, uint32_t, uint32_t>(a.w, b.w);
726 + return c;
727 +}
728 +
729 +////////////////////////////////////////////////////////////////////////////////////////////////////
730 +
731 +template<>
732 +inline __device__ uint4 mul(uint16_t a, uint4 b)
733 +{
734 + uint32_t s = h0_h0(a);
735 + uint4 c;
736 + c.x = mul<uint32_t, uint32_t, uint32_t>(s, b.x);
737 + c.y = mul<uint32_t, uint32_t, uint32_t>(s, b.y);
738 + c.z = mul<uint32_t, uint32_t, uint32_t>(s, b.z);
739 + c.w = mul<uint32_t, uint32_t, uint32_t>(s, b.w);
740 + return c;
741 +}
742 +
743 +////////////////////////////////////////////////////////////////////////////////////////////////////
744 +
745 +template<>
746 +inline __device__ float mul(uint16_t a, uint16_t b)
747 +{
748 + float fa = half_to_float(a);
749 + float fb = half_to_float(b);
750 + return fa * fb;
751 +}
752 +
753 +////////////////////////////////////////////////////////////////////////////////////////////////////
754 +
755 +template<>
756 +inline __device__ float2 mul(uint32_t a, uint32_t b)
757 +{
758 + float2 fa = half2_to_float2(a);
759 + float2 fb = half2_to_float2(b);
760 + return mul<float2, float2, float2>(fa, fb);
761 +}
762 +
763 +////////////////////////////////////////////////////////////////////////////////////////////////////
764 +
765 +template<>
766 +inline __device__ float2 mul(uint16_t a, uint32_t b)
767 +{
768 + return mul<float2, uint32_t, uint32_t>(h0_h0(a), b);
769 +}
770 +
771 +////////////////////////////////////////////////////////////////////////////////////////////////////
772 +
773 +template<>
774 +inline __device__ Float4_ mul(uint2 a, uint2 b)
775 +{
776 + Float4_ fc;
777 + fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
778 + fc.y = mul<float2, uint32_t, uint32_t>(a.y, b.y);
779 + return fc;
780 +}
781 +
782 +////////////////////////////////////////////////////////////////////////////////////////////////////
783 +
784 +template<>
785 +inline __device__ Float4_ mul(uint16_t a, uint2 b)
786 +{
787 + uint32_t s = h0_h0(a);
788 + Float4_ fc;
789 + fc.x = mul<float2, uint32_t, uint32_t>(s, b.x);
790 + fc.y = mul<float2, uint32_t, uint32_t>(s, b.y);
791 + return fc;
792 +}
793 +
794 +////////////////////////////////////////////////////////////////////////////////////////////////////
795 +
796 +template<>
797 +inline __device__ Float8_ mul(uint4 a, uint4 b)
798 +{
799 + Float8_ fc;
800 + fc.x = mul<float2, uint32_t, uint32_t>(a.x, b.x);
801 + fc.y = mul<float2, uint32_t, uint32_t>(a.y, b.y);
802 + fc.z = mul<float2, uint32_t, uint32_t>(a.z, b.z);
803 + fc.w = mul<float2, uint32_t, uint32_t>(a.w, b.w);
804 + return fc;
805 +}
806 +
807 +////////////////////////////////////////////////////////////////////////////////////////////////////
808 +
809 +template<>
810 +inline __device__ Float8_ mul(uint16_t a, uint4 b)
811 +{
812 + uint32_t s = h0_h0(a);
813 + Float8_ fc;
814 + fc.x = mul<float2, uint32_t, uint32_t>(s, b.x);
815 + fc.y = mul<float2, uint32_t, uint32_t>(s, b.y);
816 + fc.z = mul<float2, uint32_t, uint32_t>(s, b.z);
817 + fc.w = mul<float2, uint32_t, uint32_t>(s, b.w);
818 + return fc;
819 +}
820 +
821 +////////////////////////////////////////////////////////////////////////////////////////////////////
822 +
823 +#ifdef ENABLE_BF16
824 +template<>
825 +inline __device__ __nv_bfloat16 mul(__nv_bfloat16 a, __nv_bfloat16 b)
826 +{
827 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
828 + return __hmul(a, b);
829 +#else
830 + return bf16hmul(a, b);
831 +#endif
832 +}
833 +
834 +////////////////////////////////////////////////////////////////////////////////////////////////////
835 +
836 +template<>
837 +inline __device__ __nv_bfloat162 mul(__nv_bfloat162 a, __nv_bfloat162 b)
838 +{
839 + return bf16hmul2(a, b);
840 +}
841 +
842 +////////////////////////////////////////////////////////////////////////////////////////////////////
843 +
844 +template<>
845 +inline __device__ __nv_bfloat162 mul(__nv_bfloat16 a, __nv_bfloat162 b)
846 +{
847 + return mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
848 +}
849 +
850 +////////////////////////////////////////////////////////////////////////////////////////////////////
851 +
852 +template<>
853 +inline __device__ bf16_4_t mul(bf16_4_t a, bf16_4_t b)
854 +{
855 + bf16_4_t c;
856 + c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
857 + c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
858 + return c;
859 +}
860 +
861 +////////////////////////////////////////////////////////////////////////////////////////////////////
862 +
863 +template<>
864 +inline __device__ bf16_4_t mul(__nv_bfloat16 a, bf16_4_t b)
865 +{
866 + __nv_bfloat162 s = bf162bf162(a);
867 + bf16_4_t c;
868 + c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.x);
869 + c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.y);
870 + return c;
871 +}
872 +
873 +////////////////////////////////////////////////////////////////////////////////////////////////////
874 +
875 +template<>
876 +inline __device__ bf16_8_t mul(bf16_8_t a, bf16_8_t b)
877 +{
878 + bf16_8_t c;
879 + c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
880 + c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
881 + c.z = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.z, b.z);
882 + c.w = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(a.w, b.w);
883 + return c;
884 +}
885 +
886 +////////////////////////////////////////////////////////////////////////////////////////////////////
887 +
888 +template<>
889 +inline __device__ bf16_8_t mul(__nv_bfloat16 a, bf16_8_t b)
890 +{
891 + __nv_bfloat162 s = bf162bf162(a);
892 + bf16_8_t c;
893 + c.x = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.x);
894 + c.y = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.y);
895 + c.z = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.z);
896 + c.w = mul<__nv_bfloat162, __nv_bfloat162, __nv_bfloat162>(s, b.w);
897 + return c;
898 +}
899 +
900 +////////////////////////////////////////////////////////////////////////////////////////////////////
901 +
902 +template<>
903 +inline __device__ float mul(__nv_bfloat16 a, __nv_bfloat16 b)
904 +{
905 + float fa = (float)a;
906 + float fb = (float)b;
907 + return fa * fb;
908 +}
909 +
910 +////////////////////////////////////////////////////////////////////////////////////////////////////
911 +
912 +template<>
913 +inline __device__ float2 mul(__nv_bfloat162 a, __nv_bfloat162 b)
914 +{
915 + float2 fa = bf1622float2(a);
916 + float2 fb = bf1622float2(b);
917 + return mul<float2, float2, float2>(fa, fb);
918 +}
919 +
920 +////////////////////////////////////////////////////////////////////////////////////////////////////
921 +
922 +template<>
923 +inline __device__ float2 mul(__nv_bfloat16 a, __nv_bfloat162 b)
924 +{
925 + return mul<float2, __nv_bfloat162, __nv_bfloat162>(bf162bf162(a), b);
926 +}
927 +
928 +////////////////////////////////////////////////////////////////////////////////////////////////////
929 +
930 +template<>
931 +inline __device__ Float4_ mul(bf16_4_t a, bf16_4_t b)
932 +{
933 + Float4_ fc;
934 + fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
935 + fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
936 + return fc;
937 +}
938 +
939 +////////////////////////////////////////////////////////////////////////////////////////////////////
940 +
941 +template<>
942 +inline __device__ Float4_ mul(__nv_bfloat16 a, bf16_4_t b)
943 +{
944 + __nv_bfloat162 s = bf162bf162(a);
945 + Float4_ fc;
946 + fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.x);
947 + fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.y);
948 + return fc;
949 +}
950 +
951 +////////////////////////////////////////////////////////////////////////////////////////////////////
952 +
953 +template<>
954 +inline __device__ Float8_ mul(bf16_8_t a, bf16_8_t b)
955 +{
956 + Float8_ fc;
957 + fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.x, b.x);
958 + fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.y, b.y);
959 + fc.z = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.z, b.z);
960 + fc.w = mul<float2, __nv_bfloat162, __nv_bfloat162>(a.w, b.w);
961 + return fc;
962 +}
963 +
964 +////////////////////////////////////////////////////////////////////////////////////////////////////
965 +
966 +template<>
967 +inline __device__ Float8_ mul(__nv_bfloat16 a, bf16_8_t b)
968 +{
969 + __nv_bfloat162 s = bf162bf162(a);
970 + Float8_ fc;
971 + fc.x = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.x);
972 + fc.y = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.y);
973 + fc.z = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.z);
974 + fc.w = mul<float2, __nv_bfloat162, __nv_bfloat162>(s, b.w);
975 + return fc;
976 +}
977 +#endif // ENABLE_BF16
978 +////////////////////////////////////////////////////////////////////////////////////////////////////
979 +
980 +inline __device__ float sum(float v)
981 +{
982 + return v;
983 +}
984 +
985 +////////////////////////////////////////////////////////////////////////////////////////////////////
986 +
987 +inline __device__ float sum(float2 v)
988 +{
989 + return v.x + v.y;
990 +}
991 +
992 +////////////////////////////////////////////////////////////////////////////////////////////////////
993 +
994 +inline __device__ float sum(float4 v)
995 +{
996 + return v.x + v.y + v.z + v.w;
997 +}
998 +
999 +////////////////////////////////////////////////////////////////////////////////////////////////////
1000 +
1001 +#ifdef ENABLE_BF16
1002 +inline __device__ float sum(__nv_bfloat162 v)
1003 +{
1004 + float2 vf = bf1622float2(v);
1005 + return vf.x + vf.y;
1006 +}
1007 +
1008 +////////////////////////////////////////////////////////////////////////////////////////////////////
1009 +
1010 +inline __device__ float sum(bf16_4_t v)
1011 +{
1012 + return sum(v.x) + sum(v.y);
1013 +}
1014 +
1015 +////////////////////////////////////////////////////////////////////////////////////////////////////
1016 +
1017 +inline __device__ float sum(bf16_8_t v)
1018 +{
1019 + return sum(v.x) + sum(v.y) + sum(v.z) + sum(v.w);
1020 +}
1021 +#endif // ENABLE_BF16
1022 +////////////////////////////////////////////////////////////////////////////////////////////////////
1023 +
1024 +inline __device__ float sum(uint16_t v)
1025 +{
1026 + return half_to_float(v);
1027 +}
1028 +
1029 +////////////////////////////////////////////////////////////////////////////////////////////////////
1030 +
1031 +inline __device__ float sum(uint32_t v)
1032 +{
1033 + float2 tmp = half2_to_float2(v);
1034 + return tmp.x + tmp.y;
1035 +}
1036 +
1037 +////////////////////////////////////////////////////////////////////////////////////////////////////
1038 +
1039 +inline __device__ float sum(uint2 v)
1040 +{
1041 + uint32_t c = add(v.x, v.y);
1042 + return sum(c);
1043 +}
1044 +
1045 +////////////////////////////////////////////////////////////////////////////////////////////////////
1046 +
1047 +inline __device__ float sum(uint4 v)
1048 +{
1049 +#if 1
1050 + uint32_t c = add(v.x, v.y);
1051 + c = add(c, v.z);
1052 + c = add(c, v.w);
1053 +#else
1054 + uint32_t c = add(v.x, v.y);
1055 + uint32_t d = add(v.z, v.w);
1056 + c = add(c, d);
1057 +#endif
1058 + return sum(c);
1059 +}
1060 +
1061 +////////////////////////////////////////////////////////////////////////////////////////////////////
1062 +
1063 +inline __device__ float sum(Float4_ v)
1064 +{
1065 + return v.x.x + v.x.y + v.y.x + v.y.y;
1066 +}
1067 +
1068 +////////////////////////////////////////////////////////////////////////////////////////////////////
1069 +
1070 +inline __device__ float sum(Float8_ v)
1071 +{
1072 + return v.x.x + v.x.y + v.y.x + v.y.y + v.z.x + v.z.y + v.w.x + v.w.y;
1073 +}
1074 +
1075 +////////////////////////////////////////////////////////////////////////////////////////////////////
1076 +
1077 +inline __device__ float dot(Float4_ a, Float4_ b)
1078 +{
1079 + float2 acc = mul<float2, float2, float2>(a.x, b.x);
1080 + acc = fma(a.y, b.y, acc);
1081 + return acc.x + acc.y;
1082 +}
1083 +
1084 +////////////////////////////////////////////////////////////////////////////////////////////////////
1085 +
1086 +inline __device__ float dot(Float8_ a, Float8_ b)
1087 +{
1088 + float2 acc = mul<float2, float2, float2>(a.x, b.x);
1089 + acc = fma(a.y, b.y, acc);
1090 + acc = fma(a.z, b.z, acc);
1091 + acc = fma(a.w, b.w, acc);
1092 + return acc.x + acc.y;
1093 +}
1094 +
1095 +////////////////////////////////////////////////////////////////////////////////////////////////////
1096 +
1097 +template<typename T>
1098 +inline __device__ float dot(T a, T b)
1099 +{
1100 + return sum(mul<T, T, T>(a, b));
1101 +}
1102 +
1103 +////////////////////////////////////////////////////////////////////////////////////////////////////
1104 +
1105 +template<typename A, typename T>
1106 +inline __device__ float dot(T a, T b)
1107 +{
1108 + return sum(mul<A, T, T>(a, b));
1109 +}
1110 +
1111 +////////////////////////////////////////////////////////////////////////////////////////////////////
1112 +
1113 +inline __device__ void zero(uint16_t& dst)
1114 +{
1115 + dst = uint16_t(0);
1116 +}
1117 +
1118 +////////////////////////////////////////////////////////////////////////////////////////////////////
1119 +
1120 +template<typename T>
1121 +inline __device__ void zero(T& dst)
1122 +{
1123 + constexpr int WORDS = sizeof(T) / 4;
1124 + union {
1125 + T raw;
1126 + uint32_t words[WORDS];
1127 + } tmp;
1128 +#pragma unroll
1129 + for (int ii = 0; ii < WORDS; ++ii) {
1130 + tmp.words[ii] = 0u;
1131 + }
1132 + dst = tmp.raw;
1133 +}
1134 +
1135 +
1136 +////////////////////////////////////////////////////////////////////////////////////////////////////
1137 +
1138 +inline __device__ void convert_from_float(float& dst, float src)
1139 +{
1140 + dst = src;
1141 +}
1142 +
1143 +////////////////////////////////////////////////////////////////////////////////////////////////////
1144 +
1145 +inline __device__ void convert_from_float(uint16_t& dst, float src)
1146 +{
1147 + dst = float_to_half(src);
1148 +}
1149 +
1150 +////////////////////////////////////////////////////////////////////////////////////////////////////
1151 +
1152 +inline __device__ void convert_from_float(uint32_t& dst, float2 src)
1153 +{
1154 + dst = float2_to_half2(src);
1155 +}
1156 +
1157 +////////////////////////////////////////////////////////////////////////////////////////////////////
1158 +#ifdef ENABLE_BF16
1159 +inline __device__ void convert_from_float(__nv_bfloat16& dst, float src)
1160 +{
1161 + dst = __float2bfloat16(src);
1162 +}
1163 +
1164 +////////////////////////////////////////////////////////////////////////////////////////////////////
1165 +
1166 +inline __device__ void convert_from_float(__nv_bfloat162& dst, float2 src)
1167 +{
1168 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1169 + dst = __float22bfloat162_rn(src);
1170 +#else
1171 + dst = __floats2bfloat162_rn(src.x, src.y);
1172 +#endif
1173 +}
1174 +#endif // ENABLE_BF16
1175 +////////////////////////////////////////////////////////////////////////////////////////////////////
1176 +
1177 +inline __device__ void convert_from_float(uint2& dst, Float4_ src)
1178 +{
1179 + dst.x = float2_to_half2(src.x);
1180 + dst.y = float2_to_half2(src.y);
1181 +}
1182 +
1183 +////////////////////////////////////////////////////////////////////////////////////////////////////
1184 +
1185 +inline __device__ void convert_from_float(uint4& dst, Float8_ src)
1186 +{
1187 + dst.x = float2_to_half2(src.x);
1188 + dst.y = float2_to_half2(src.y);
1189 + dst.z = float2_to_half2(src.z);
1190 + dst.w = float2_to_half2(src.w);
1191 +}
1192 +
1193 +////////////////////////////////////////////////////////////////////////////////////////////////////
1194 +
1195 +#ifdef ENABLE_BF16
1196 +inline __device__ void convert_from_float(bf16_4_t& dst, Float4_ src)
1197 +{
1198 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1199 + dst.x = __float22bfloat162_rn(src.x);
1200 + dst.y = __float22bfloat162_rn(src.y);
1201 +#else
1202 + dst.x = __floats2bfloat162_rn(src.x.x, src.x.y);
1203 + dst.y = __floats2bfloat162_rn(src.y.x, src.y.y);
1204 +#endif
1205 +}
1206 +
1207 +////////////////////////////////////////////////////////////////////////////////////////////////////
1208 +
1209 +inline __device__ void convert_from_float(bf16_8_t& dst, Float8_ src)
1210 +{
1211 +#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 800
1212 + dst.x = __float22bfloat162_rn(src.x);
1213 + dst.y = __float22bfloat162_rn(src.y);
1214 + dst.z = __float22bfloat162_rn(src.z);
1215 + dst.w = __float22bfloat162_rn(src.w);
1216 +#else
1217 + dst.x = __floats2bfloat162_rn(src.x.x, src.x.y);
1218 + dst.y = __floats2bfloat162_rn(src.y.x, src.y.y);
1219 + dst.z = __floats2bfloat162_rn(src.z.x, src.z.y);
1220 + dst.w = __floats2bfloat162_rn(src.w.x, src.w.y);
1221 +#endif
1222 +}
1223 +#endif // ENABLE_BF16
1224 +
1225 +////////////////////////////////////////////////////////////////////////////////////////////////////
1226 +
1227 +inline __device__ void convert_from_float(float2& dst, float2 src)
1228 +{
1229 + dst = src;
1230 +}
1231 +
1232 +////////////////////////////////////////////////////////////////////////////////////////////////////
1233 +
1234 +inline __device__ void convert_from_float(float4& dst, float4 src)
1235 +{
1236 + dst = src;
1237 +}
1238 +
1239 +////////////////////////////////////////////////////////////////////////////////////////////////////
1240 +
1241 +inline __device__ float convert_to_float(float4 u)
1242 +{
1243 + return u.x;
1244 +}
1245 +
1246 +////////////////////////////////////////////////////////////////////////////////////////////////////
1247 +
1248 +inline __device__ float convert_to_float(uint4 u)
1249 +{
1250 + float2 tmp = half2_to_float2(u.x);
1251 + return tmp.x;
1252 +}
1253 +
1254 +////////////////////////////////////////////////////////////////////////////////////////////////////
1255 +
1256 +inline __device__ float cast_to_float(float u)
1257 +{
1258 + return u;
1259 +}
1260 +
1261 +////////////////////////////////////////////////////////////////////////////////////////////////////
1262 +
1263 +inline __device__ float2 cast_to_float(float2 u)
1264 +{
1265 + return u;
1266 +}
1267 +
1268 +////////////////////////////////////////////////////////////////////////////////////////////////////
1269 +
1270 +inline __device__ float4 cast_to_float(float4 u)
1271 +{
1272 + return u;
1273 +}
1274 +
1275 +////////////////////////////////////////////////////////////////////////////////////////////////////
1276 +
1277 +inline __device__ Float4_ cast_to_float(Float4_ u)
1278 +{
1279 + return u;
1280 +}
1281 +
1282 +////////////////////////////////////////////////////////////////////////////////////////////////////
1283 +
1284 +inline __device__ Float8_ cast_to_float(Float8_ u)
1285 +{
1286 + return u;
1287 +}
1288 +
1289 +////////////////////////////////////////////////////////////////////////////////////////////////////
1290 +
1291 +inline __device__ float2 cast_to_float(uint32_t u)
1292 +{
1293 + return half2_to_float2(u);
1294 +}
1295 +
1296 +////////////////////////////////////////////////////////////////////////////////////////////////////
1297 +
1298 +inline __device__ Float4_ cast_to_float(uint2 u)
1299 +{
1300 + Float4_ tmp;
1301 + tmp.x = half2_to_float2(u.x);
1302 + tmp.y = half2_to_float2(u.y);
1303 + return tmp;
1304 +}
1305 +
1306 +////////////////////////////////////////////////////////////////////////////////////////////////////
1307 +
1308 +inline __device__ Float8_ cast_to_float(uint4 u)
1309 +{
1310 + Float8_ tmp;
1311 + tmp.x = half2_to_float2(u.x);
1312 + tmp.y = half2_to_float2(u.y);
1313 + tmp.z = half2_to_float2(u.z);
1314 + tmp.w = half2_to_float2(u.w);
1315 + return tmp;
1316 +}
1317 +
1318 +}
setup.py
+9 -2
@@ -9,15 +9,22 @@ ext_modules = []
9
10 # Cache operations.
11 cache_extension = cpp_extension.CUDAExtension(
12 - name='cacheflow.ops',
12 + name='cacheflow.cache_ops',
13 sources=['csrc/cache.cpp', 'csrc/cache_kernels.cu'],
14 extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
15 )
16 ext_modules.append(cache_extension)
17
18 +# Attention kernels.
19 +attention_extension = cpp_extension.CUDAExtension(
20 + name='cacheflow.attention_ops',
21 + sources=['csrc/attention.cpp', 'csrc/attention_kernels.cu'],
22 + extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
23 +)
24 +ext_modules.append(attention_extension)
25 +
26 setuptools.setup(
27 name='cacheflow',
20 - requires_python='>=3.9',
28 ext_modules=ext_modules,
29 cmdclass={'build_ext': cpp_extension.BuildExtension},
30 )
tests/kernels/attention.py new
+142
@@ -0,0 +1,142 @@
1 +import random
2 +from typing import Optional
3 +
4 +import torch
5 +
6 +from cacheflow import attention_ops
7 +
8 +
9 +def ref_masked_attention(
10 + query: torch.Tensor,
11 + key: torch.Tensor,
12 + value: torch.Tensor,
13 + scale: float,
14 + attn_mask: Optional[torch.Tensor] = None,
15 +) -> torch.Tensor:
16 + query = query * scale
17 + attn = torch.einsum('qhd,khd->hqk', query, key)
18 + if attn_mask is not None:
19 + attn = attn + attn_mask
20 + attn = torch.softmax(attn, dim=-1)
21 + out = torch.einsum('hqk,khd->qhd', attn, value)
22 + return out
23 +
24 +
25 +def ref_single_query_cached_kv_attention(
26 + output: torch.Tensor,
27 + query: torch.Tensor,
28 + key_cache: torch.Tensor,
29 + value_cache: torch.Tensor,
30 + block_tables: torch.Tensor,
31 + context_lens: torch.Tensor,
32 +) -> None:
33 + num_heads = value_cache.shape[1]
34 + head_size = value_cache.shape[2]
35 + block_size = value_cache.shape[3]
36 +
37 + num_input_tokens = query.shape[0]
38 + for i in range(num_input_tokens):
39 + q = query[i].unsqueeze(0)
40 + block_table = block_tables[i]
41 + context_len = int(context_lens[i])
42 +
43 + keys = []
44 + values = []
45 + for j in range(context_len):
46 + block_number = int(block_table[j // block_size])
47 + block_offset = j % block_size
48 +
49 + k = key_cache[block_number, :, :, block_offset, :]
50 + k = k.reshape(num_heads, head_size)
51 + keys.append(k)
52 +
53 + v = value_cache[block_number, :, :, block_offset]
54 + values.append(v)
55 + keys = torch.stack(keys, dim=0)
56 + values = torch.stack(values, dim=0)
57 +
58 + scale = 1.0 / (head_size ** 0.5)
59 + out = ref_masked_attention(q, keys, values, scale)
60 + out = out.view(num_heads, head_size)
61 + output[i].copy_(out, non_blocking=True)
62 +
63 +
64 +def test_single_query_cached_kv_attention(
65 + num_tokens: int,
66 + num_heads: int,
67 + head_size: int,
68 + block_size: int,
69 + num_blocks: int,
70 + dtype: torch.dtype,
71 +) -> None:
72 + query = torch.randn(
73 + num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
74 + x = 16 // torch.tensor([], dtype=dtype).element_size()
75 + key_block_shape = (num_heads, head_size // x, block_size, x)
76 + key_cache = torch.randn(
77 + size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
78 + value_block_shape = (num_heads, head_size, block_size)
79 + value_cache = torch.randn(
80 + size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
81 +
82 + context_lens = [random.randint(1, 4096) for _ in range(num_tokens)]
83 + max_context_len = max(context_lens)
84 + context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
85 +
86 + max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
87 + block_tables = []
88 + for _ in range(num_tokens):
89 + block_table = [
90 + random.randint(0, num_blocks - 1)
91 + for _ in range(max_num_blocks_per_seq)
92 + ]
93 + block_tables.append(block_table)
94 + block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
95 +
96 + scale = float(1.0 / (head_size ** 0.5))
97 + output = torch.empty_like(query)
98 + attention_ops.single_query_cached_kv_attention(
99 + output,
100 + query,
101 + key_cache,
102 + value_cache,
103 + scale,
104 + block_tables,
105 + context_lens,
106 + block_size,
107 + max_context_len,
108 + )
109 +
110 + ref_output = torch.empty_like(query)
111 + ref_single_query_cached_kv_attention(
112 + ref_output,
113 + query,
114 + key_cache,
115 + value_cache,
116 + block_tables,
117 + context_lens,
118 + )
119 + # NOTE(woosuk): Due to the difference in the data types the two
120 + # implementations use for attention softmax logits and accumulation,
121 + # there is a small difference in the final outputs.
122 + # We should use a relaxed tolerance for the test.
123 + assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
124 +
125 +
126 +@torch.inference_mode()
127 +def test_attention() -> None:
128 + for dtype in [torch.half, torch.float]:
129 + for block_size in [8, 16]:
130 + for head_size in [64, 80, 96, 128, 256]:
131 + test_single_query_cached_kv_attention(
132 + num_tokens=37,
133 + num_heads=3,
134 + head_size=head_size,
135 + block_size=block_size,
136 + num_blocks=1024,
137 + dtype=dtype,
138 + )
139 +
140 +
141 +if __name__ == '__main__':
142 + test_attention()
tests/kernels/cache.py renamed
+8 -8
@@ -2,7 +2,7 @@ import random
2
3 import torch
4
5 -from cacheflow.ops import reshape_and_cache
5 +from cacheflow import cache_ops
6
7
8 def test_reshape_and_cache(
@@ -26,30 +26,30 @@ def test_reshape_and_cache(
26 key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
27 cloned_key_cache = key_cache.clone()
28
29 - value_cache_shape = (num_blocks, num_heads, block_size, head_size)
29 + value_cache_shape = (num_blocks, num_heads, head_size, block_size)
30 value_cache = torch.randn(
31 size=value_cache_shape, dtype=dtype, device='cuda')
32 cloned_value_cache = value_cache.clone()
33
34 - reshape_and_cache(key, value, key_cache, value_cache, slot_mapping)
34 + cache_ops.reshape_and_cache(key, value, key_cache, value_cache, slot_mapping)
35
36 for i in range(num_tokens):
37 reshaped_key = key.reshape(num_tokens, num_heads, head_size // x, x)
38 block_idx = slot_mapping[i] // block_size
39 block_offset = slot_mapping[i] % block_size
40 cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
41 - cloned_value_cache[block_idx, :, block_offset, :] = value[i]
41 + cloned_value_cache[block_idx, :, :, block_offset] = value[i]
42
43 assert torch.allclose(key_cache, cloned_key_cache)
44 assert torch.allclose(value_cache, cloned_value_cache)
45
46
47 -@torch.no_grad()
48 -def test_kernels():
47 +@torch.inference_mode()
48 +def test_cache() -> None:
49 test_reshape_and_cache(
50 - num_tokens=3, num_heads=2, head_size=16, block_size=2, num_blocks=2,
50 + num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
51 dtype=torch.half)
52
53
54 if __name__ == '__main__':
55 - test_kernels()
55 + test_cache()