@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0f4b32199e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Support various block sizes & Change default block size to 16 (#38)

Woosuk Kwon committed Apr 15, 2023 at 09:03 UTC 0f4b32199ec6c5d16bc03767e36fff2d54559ff8
7 files changed +602 -619
benchmark/benchmark_text_completion.py
+1
@@ -268,6 +268,7 @@ if __name__ == '__main__':
268 f'{model_name}-tp{args.tensor_parallel_size}',
269 sample_dir,
270 'cacheflow',
271 + f'block{args.block_size}',
272 f'req-rate-{args.request_rate}',
273 f'seed{args.seed}',
274 f'duration-{args.duration}',
cacheflow/master/block_manager.py
-3
@@ -15,9 +15,6 @@ class BlockAllocator:
15 block_size: int,
16 num_blocks: int,
17 ) -> None:
18 - if block_size not in [8, 16, 32]:
19 - raise ValueError(f'Unsupported block size: {block_size}'
20 - 'The block size must be one of {8, 16, 32}.')
18 self.device = device
19 self.block_size = block_size
20 self.num_blocks = num_blocks
cacheflow/master/scheduler.py
+2 -1
@@ -125,7 +125,8 @@ class Scheduler:
125
126 # Swap in the sequence groups in the SWAPPED state if possible.
127 self.swapped = self.policy.sort_by_priority(now, self.swapped)
128 - while self.swapped:
128 + # FCFS
129 + while self.swapped and not blocks_to_swap_out:
130 seq_group = self.swapped[0]
131 # If the sequence group has been preempted in this step, stop.
132 if seq_group in preempted:
cacheflow/master/server.py
+2 -2
@@ -180,9 +180,9 @@ def add_server_arguments(parser: argparse.ArgumentParser):
180 parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
181 parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
182 # KV cache arguments
183 - parser.add_argument('--block-size', type=int, default=8, choices=[8, 16, 32], help='token block size')
183 + parser.add_argument('--block-size', type=int, default=16, choices=[1, 2, 4, 8, 16, 32, 64, 128, 256], help='token block size')
184 # NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
185 - parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
185 + parser.add_argument('--dtype', type=str, default='half', choices=['half'], help='data type')
186 # TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
187 parser.add_argument('--seed', type=int, default=0, help='random seed')
188 parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
csrc/attention.cpp
-16
@@ -11,25 +11,9 @@ void single_query_cached_kv_attention(
11 int block_size,
12 int max_context_len);
13
14 -void multi_query_cached_kv_attention(
15 - torch::Tensor& cu_query_lens,
16 - torch::Tensor& out,
17 - torch::Tensor& query,
18 - torch::Tensor& key_cache,
19 - torch::Tensor& value_cache,
20 - float scale,
21 - torch::Tensor& block_tables,
22 - torch::Tensor& context_lens,
23 - int block_size,
24 - int max_context_len);
25 -
14 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
15 m.def(
16 "single_query_cached_kv_attention",
17 &single_query_cached_kv_attention,
18 "Compute the attention between an input query and the cached key/value tensors");
31 - m.def(
32 - "multi_query_cached_kv_attention",
33 - &multi_query_cached_kv_attention,
34 - "Compute the attention between multiple input queries and the cached key/value tensors");
19 }
csrc/attention_kernels.cu
+557 -579
@@ -8,6 +8,8 @@
8 #include <algorithm>
9
10 #define WARP_SIZE 32
11 +#define MAX(a, b) ((a) > (b) ? (a) : (b))
12 +#define MIN(a, b) ((a) < (b) ? (a) : (b))
13
14 namespace cacheflow {
15
@@ -27,7 +29,8 @@ __global__ void single_query_cached_kv_attention_kernel(
29 const int* __restrict__ context_lens, // [num_seqs]
30 const int max_num_blocks_per_seq,
31 const int q_stride) {
30 - constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
32 + constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
33 + constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
34 constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
35 const int thread_idx = threadIdx.x;
36 const int warp_idx = thread_idx / WARP_SIZE;
@@ -39,10 +42,10 @@ __global__ void single_query_cached_kv_attention_kernel(
42
43 // A vector type to store a part of a key or a query.
44 // The vector size is configured in such a way that the threads in a thread group
42 - // fetch or comput 16 bytes at a time.
45 + // fetch or compute 16 bytes at a time.
46 // For example, if the size of a thread group is 4 and the data type is half,
47 // then the vector size is 16 / (4 * sizeof(half)) == 2.
45 - constexpr int VEC_SIZE = 16 / (THREAD_GROUP_SIZE * sizeof(scalar_t));
48 + constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
49 using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
50 using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
51
@@ -88,284 +91,42 @@ __global__ void single_query_cached_kv_attention_kernel(
91 // dot product with the query.
92 for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
93 const int physical_block_number = block_table[block_idx];
91 - const int physical_block_offset = thread_group_idx % BLOCK_SIZE;
92 - const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
94
95 // Load a key to registers.
96 // Each thread in a thread group has a different part of the key.
97 // For example, if the the thread group size is 4, then the first thread in the group
98 // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
99 // vectors of the key, and so on.
99 - K_vec k_vecs[NUM_VECS_PER_THREAD];
100 -#pragma unroll
101 - for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
102 - const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
103 - + head_idx * HEAD_SIZE * BLOCK_SIZE
104 - + physical_block_offset * x;
105 - const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
106 - const int offset1 = (vec_idx * VEC_SIZE) / x;
107 - const int offset2 = (vec_idx * VEC_SIZE) % x;
108 - k_vecs[i] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
109 - }
110 -
111 - // Compute dot product.
112 - // This includes a reduction across the threads in the same thread group.
113 - const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
114 - const bool mask = token_idx >= context_len;
115 -
116 - if (thread_group_offset == 0) {
117 - // Store the partial reductions to shared memory.
118 - // NOTE(woosuk): It is required to zero out the masked logits.
119 - logits[token_idx] = mask ? 0.f : qk;
120 - // Update the max value.
121 - qk_max = mask ? qk_max : fmaxf(qk_max, qk);
122 - }
123 - }
124 -
125 - // Perform reduction across the threads in the same warp to get the
126 - // max qk value for each "warp" (not across the thread block yet).
127 - // The 0-th thread of each thread group already has its max qk value.
128 -#pragma unroll
129 - for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
130 - qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
131 - }
132 - if (lane == 0) {
133 - red_smem[warp_idx] = qk_max;
134 - }
135 - __syncthreads();
136 -
137 - // TODO(woosuk): Refactor this part.
138 - // Get the max qk value for the sequence.
139 - qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
140 -#pragma unroll
141 - for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
142 - qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
143 - }
144 - // Broadcast the max qk value to all threads.
145 - qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
146 -
147 - // Get the sum of the exp values.
148 - float exp_sum = 0.f;
149 - for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
150 - float val = __expf(logits[i] - qk_max);
151 - logits[i] = val;
152 - exp_sum += val;
153 - }
154 - exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
155 -
156 - // Compute softmax.
157 - const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
158 - for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
159 - logits[i] *= inv_sum;
160 - }
161 - __syncthreads();
162 -
163 - // Each thread will fetch 16 bytes from the value cache at a time.
164 - constexpr int V_VEC_SIZE = 16 / sizeof(scalar_t);
165 - using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
166 - using L_vec = typename FloatVec<V_vec>::Type;
167 -
168 - constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
169 - constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
170 - constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
171 -
172 - float accs[NUM_ROWS_PER_THREAD];
173 -#pragma unroll
174 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
175 - accs[i] = 0.f;
176 - }
177 -
178 - for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
179 - const int physical_block_number = block_table[block_idx];
180 - const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
181 - const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
182 - L_vec logits_vec = *reinterpret_cast<L_vec*>(logits + token_idx);
183 -
184 - const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
185 - + head_idx * HEAD_SIZE * BLOCK_SIZE;
186 -#pragma unroll
187 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
188 - const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
189 - if (row_idx < HEAD_SIZE) {
190 - const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
191 - V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
192 - accs[i] += dot(logits_vec, cast_to_float(v_vec));
193 - }
194 - }
195 - }
196 -
197 - // Perform reduction within each warp.
198 -#pragma unroll
199 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
200 - float acc = accs[i];
201 -#pragma unroll
202 - for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
203 - acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
204 - }
205 - accs[i] = acc;
206 - }
100 + for (int i = 0; i < NUM_TOKENS_PER_THREAD_GROUP; i++) {
101 + const int physical_block_offset = (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;
102 + const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
103 + K_vec k_vecs[NUM_VECS_PER_THREAD];
104
208 - // NOTE(woosuk): A barrier is required because the shared memory space for logits
209 - // is reused for the output.
210 - __syncthreads();
211 -
212 - // Perform reduction across warps.
213 - float* out_smem = reinterpret_cast<float*>(shared_mem);
214 -#pragma unroll
215 - for (int i = NUM_WARPS; i > 1; i /= 2) {
216 - int mid = i / 2;
217 - // Upper warps write to shared memory.
218 - if (warp_idx >= mid && warp_idx < i) {
219 - float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
105 #pragma unroll
221 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
222 - const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
223 - if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
224 - dst[row_idx] = accs[i];
225 - }
106 + for (int j = 0; j < NUM_VECS_PER_THREAD; j++) {
107 + const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
108 + + head_idx * HEAD_SIZE * BLOCK_SIZE
109 + + physical_block_offset * x;
110 + const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
111 + const int offset1 = (vec_idx * VEC_SIZE) / x;
112 + const int offset2 = (vec_idx * VEC_SIZE) % x;
113 + k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
114 }
227 - }
228 - __syncthreads();
115
230 - // Lower warps update the output.
231 - if (warp_idx < mid) {
232 - const float* src = &out_smem[warp_idx * HEAD_SIZE];
233 -#pragma unroll
234 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
235 - const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
236 - if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
237 - accs[i] += src[row_idx];
238 - }
239 - }
240 - }
241 - __syncthreads();
242 - }
243 -
244 - // Write the final output.
245 - if (warp_idx == 0) {
246 - scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
247 -#pragma unroll
248 - for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
249 - const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
250 - if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
251 - convert_from_float(*(out_ptr + row_idx), accs[i]);
116 + // Compute dot product.
117 + // This includes a reduction across the threads in the same thread group.
118 + const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
119 + const bool mask = token_idx >= context_len;
120 +
121 + if (thread_group_offset == 0) {
122 + // Store the partial reductions to shared memory.
123 + // NOTE(woosuk): It is required to zero out the masked logits.
124 + logits[token_idx] = mask ? 0.f : qk;
125 + // Update the max value.
126 + qk_max = mask ? qk_max : fmaxf(qk_max, qk);
127 }
128 }
129 }
255 -}
256 -
257 -
258 -// Grid: (num_heads, num_query_tokens).
259 -template<
260 - typename scalar_t,
261 - int HEAD_SIZE,
262 - int BLOCK_SIZE,
263 - int NUM_THREADS>
264 -__device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
265 - scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
266 - const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
267 - const int seq_start_idx,
268 - const int seq_len,
269 - const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
270 - const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
271 - const float scale,
272 - const int* __restrict__ block_table, // [num_seqs, max_num_blocks_per_seq]
273 - const int context_len,
274 - const int max_num_blocks_per_seq,
275 - const int q_stride) {
276 - constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
277 - constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
278 - const int thread_idx = threadIdx.x;
279 - const int warp_idx = thread_idx / WARP_SIZE;
280 - const int lane = thread_idx % WARP_SIZE;
281 -
282 - const int head_idx = blockIdx.x;
283 - const int num_heads = gridDim.x;
284 - const int seq_idx = blockIdx.y;
285 -
286 - // A vector type to store a part of a key or a query.
287 - // The vector size is configured in such a way that the threads in a thread group
288 - // fetch or comput 16 bytes at a time.
289 - // For example, if the size of a thread group is 4 and the data type is half,
290 - // then the vector size is 16 / (4 * sizeof(half)) == 2.
291 - constexpr int VEC_SIZE = 16 / (THREAD_GROUP_SIZE * sizeof(scalar_t));
292 - using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
293 - using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
294 -
295 - constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
296 - constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
297 -
298 - const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
299 - const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
300 -
301 - // Load the query to registers.
302 - // Each thread in a thread group has a different part of the query.
303 - // For example, if the the thread group size is 4, then the first thread in the group
304 - // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
305 - // th vectors of the query, and so on.
306 - // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
307 - const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
308 - Q_vec q_vecs[NUM_VECS_PER_THREAD];
309 -#pragma unroll
310 - for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
311 - const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
312 - q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
313 - }
314 -
315 - // Memory planning.
316 - extern __shared__ char shared_mem[];
317 - // NOTE(woosuk): We use FP32 logits and accumulation.
318 - float *logits = reinterpret_cast<float*>(shared_mem);
319 - // Workspace for reduction.
320 - __shared__ float red_smem[2 * NUM_WARPS];
321 -
322 - // x == THREAD_GROUP_SIZE * VEC_SIZE
323 - // Each thread group fetches x elements from the key at a time.
324 - constexpr int x = 16 / sizeof(scalar_t);
325 - float qk_max = -FLT_MAX;
326 -
327 - const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
328 - const int mask_boundary = context_len - seq_len + 1 + (seq_idx - seq_start_idx);
329 -
330 - // Iterate over the key blocks.
331 - // Each warp fetches a block of keys for each iteration.
332 - // Each thread group in a warp fetches a key from the block, and computes
333 - // dot product with the query.
334 - for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
335 - const int physical_block_number = block_table[block_idx];
336 - const int physical_block_offset = thread_group_idx % BLOCK_SIZE;
337 - const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
338 -
339 - // Load a key to registers.
340 - // Each thread in a thread group has a different part of the key.
341 - // For example, if the the thread group size is 4, then the first thread in the group
342 - // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
343 - // vectors of the key, and so on.
344 - K_vec k_vecs[NUM_VECS_PER_THREAD];
345 -#pragma unroll
346 - for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
347 - const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
348 - + head_idx * HEAD_SIZE * BLOCK_SIZE
349 - + physical_block_offset * x;
350 - const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
351 - const int offset1 = (vec_idx * VEC_SIZE) / x;
352 - const int offset2 = (vec_idx * VEC_SIZE) % x;
353 - k_vecs[i] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
354 - }
355 -
356 - // Compute dot product.
357 - // This includes a reduction across the threads in the same thread group.
358 - const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
359 - const bool mask = token_idx >= mask_boundary;
360 -
361 - if (thread_group_offset == 0) {
362 - // Store the partial reductions to shared memory.
363 - // NOTE(woosuk): It is required to zero out the masked logits.
364 - logits[token_idx] = mask ? 0.f : qk;
365 - // Update the max value.
366 - qk_max = mask ? qk_max : fmaxf(qk_max, qk);
367 - }
368 - }
130
131 // Perform reduction across the threads in the same warp to get the
132 // max qk value for each "warp" (not across the thread block yet).
@@ -391,7 +152,7 @@ __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
152
153 // Get the sum of the exp values.
154 float exp_sum = 0.f;
394 - for (int i = thread_idx; i < mask_boundary; i += NUM_THREADS) {
155 + for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
156 float val = __expf(logits[i] - qk_max);
157 logits[i] = val;
158 exp_sum += val;
@@ -406,7 +167,7 @@ __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
167 __syncthreads();
168
169 // Each thread will fetch 16 bytes from the value cache at a time.
409 - constexpr int V_VEC_SIZE = 16 / sizeof(scalar_t);
170 + constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
171 using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
172 using L_vec = typename FloatVec<V_vec>::Type;
173
@@ -499,46 +260,6 @@ __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
260 }
261 }
262
502 -
503 -// Grid: (num_heads, num_query_tokens).
504 -template<
505 - typename scalar_t,
506 - int HEAD_SIZE,
507 - int BLOCK_SIZE,
508 - int NUM_THREADS>
509 -__global__ void multi_query_cached_kv_attention_kernel(
510 - const int* cu_query_lens, // [num_prompts+1]
511 - const int* seq_prompt_mapping, // [num_seqs] mapping from seq_idx to prompt_idx
512 - scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
513 - const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
514 - const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
515 - const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
516 - const float scale,
517 - const int* __restrict__ block_tables, // [num_prompts, max_num_blocks_per_seq]
518 - const int* __restrict__ context_lens, // [num_prompts]
519 - const int max_num_blocks_per_seq,
520 - const int q_stride) {
521 - const int seq_idx = blockIdx.y;
522 - const int prompt_idx = seq_prompt_mapping[seq_idx];
523 - const int seq_start_idx = cu_query_lens[prompt_idx];
524 - const int seq_len = cu_query_lens[prompt_idx + 1] - seq_start_idx;
525 - const int* block_table = block_tables + prompt_idx * max_num_blocks_per_seq;
526 - const int context_len = context_lens[prompt_idx];
527 - multi_query_cached_kv_attention_kernel_unoptimized_<
528 - scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
529 - out,
530 - q,
531 - seq_start_idx,
532 - seq_len,
533 - k_cache,
534 - v_cache,
535 - scale,
536 - block_table,
537 - context_len,
538 - max_num_blocks_per_seq,
539 - q_stride);
540 -}
541 -
263 } // namespace cacheflow
264
265 #define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
@@ -574,6 +295,9 @@ void single_query_cached_kv_attention_launcher(
295 int max_num_blocks_per_seq = block_tables.size(1);
296 int query_stride = query.stride(0);
297
298 + int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
299 + assert(head_size % thread_group_size == 0);
300 +
301 T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
302 T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
303 T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
@@ -621,6 +345,17 @@ void single_query_cached_kv_attention_launcher(
345 }
346 }
347
348 +#define CALL_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
349 + single_query_cached_kv_attention_launcher<T, BLOCK_SIZE>( \
350 + out, \
351 + query, \
352 + key_cache, \
353 + value_cache, \
354 + scale, \
355 + block_tables, \
356 + context_lens, \
357 + max_context_len);
358 +
359 void single_query_cached_kv_attention(
360 torch::Tensor& out, // [num_seqs, num_heads, head_size]
361 torch::Tensor& query, // [num_seqs, num_heads, head_size]
@@ -634,285 +369,528 @@ void single_query_cached_kv_attention(
369 // TODO(woosuk): Support BF16.
370 if (query.element_size() == 2) {
371 // Half.
637 - if (block_size == 8) {
638 - single_query_cached_kv_attention_launcher<uint16_t, 8>(
639 - out,
640 - query,
641 - key_cache,
642 - value_cache,
643 - scale,
644 - block_tables,
645 - context_lens,
646 - max_context_len);
647 - } else if (block_size == 16) {
648 - single_query_cached_kv_attention_launcher<uint16_t, 16>(
649 - out,
650 - query,
651 - key_cache,
652 - value_cache,
653 - scale,
654 - block_tables,
655 - context_lens,
656 - max_context_len);
657 - } else if (block_size == 32) {
658 - single_query_cached_kv_attention_launcher<uint16_t, 32>(
659 - out,
660 - query,
661 - key_cache,
662 - value_cache,
663 - scale,
664 - block_tables,
665 - context_lens,
666 - max_context_len);
667 - } else {
668 - assert(false);
669 - }
670 - } else if (query.element_size() == 4) {
671 - // Float.
672 - if (block_size == 8) {
673 - single_query_cached_kv_attention_launcher<float, 8>(
674 - out,
675 - query,
676 - key_cache,
677 - value_cache,
678 - scale,
679 - block_tables,
680 - context_lens,
681 - max_context_len);
372 + if (block_size == 1) {
373 + CALL_KERNEL_LAUNCHER(uint16_t, 1);
374 + } else if (block_size == 2) {
375 + CALL_KERNEL_LAUNCHER(uint16_t, 2);
376 + } else if (block_size == 4) {
377 + CALL_KERNEL_LAUNCHER(uint16_t, 4);
378 + } else if (block_size == 8) {
379 + CALL_KERNEL_LAUNCHER(uint16_t, 8);
380 } else if (block_size == 16) {
683 - single_query_cached_kv_attention_launcher<float, 16>(
684 - out,
685 - query,
686 - key_cache,
687 - value_cache,
688 - scale,
689 - block_tables,
690 - context_lens,
691 - max_context_len);
381 + CALL_KERNEL_LAUNCHER(uint16_t, 16);
382 } else if (block_size == 32) {
693 - single_query_cached_kv_attention_launcher<float, 32>(
694 - out,
695 - query,
696 - key_cache,
697 - value_cache,
698 - scale,
699 - block_tables,
700 - context_lens,
701 - max_context_len);
383 + CALL_KERNEL_LAUNCHER(uint16_t, 32);
384 + } else if (block_size == 64) {
385 + CALL_KERNEL_LAUNCHER(uint16_t, 64);
386 + } else if (block_size == 128) {
387 + CALL_KERNEL_LAUNCHER(uint16_t, 128);
388 + } else if (block_size == 256) {
389 + CALL_KERNEL_LAUNCHER(uint16_t, 256);
390 } else {
391 assert(false);
392 }
393 } else {
394 + // Float.
395 assert(false);
396 }
397 }
398
710 -
711 -#define LAUNCH_MULTI_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
712 - cacheflow::multi_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
713 - <<<grid, block, shared_mem_size, stream>>>( \
714 - cu_query_lens_ptr, \
715 - seq_prompt_mapping_ptr, \
716 - out_ptr, \
717 - query_ptr, \
718 - key_cache_ptr, \
719 - value_cache_ptr, \
720 - scale, \
721 - block_tables_ptr, \
722 - context_lens_ptr, \
723 - max_num_blocks_per_seq, \
724 - query_stride);
725 -
726 -
727 -// TODO(woosuk): Tune NUM_THREADS.
728 -template<
729 - typename T,
730 - int BLOCK_SIZE,
731 - int NUM_THREADS = 128>
732 -void multi_query_cached_kv_attention_launcher(
733 - torch::Tensor& cu_query_lens,
734 - torch::Tensor& seq_prompt_mapping,
735 - torch::Tensor& out,
736 - torch::Tensor& query,
737 - torch::Tensor& key_cache,
738 - torch::Tensor& value_cache,
739 - float scale,
740 - torch::Tensor& block_tables,
741 - torch::Tensor& context_lens,
742 - int max_context_len) {
743 - int num_seqs = query.size(0);
744 - int num_heads = query.size(1);
745 - int head_size = query.size(2);
746 - int max_num_blocks_per_seq = block_tables.size(1);
747 - int query_stride = query.stride(0);
748 -
749 - int* cu_query_lens_ptr = cu_query_lens.data_ptr<int>();
750 - int* seq_prompt_mapping_ptr = seq_prompt_mapping.data_ptr<int>();
751 - T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
752 - T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
753 - T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
754 - T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
755 - int* block_tables_ptr = block_tables.data_ptr<int>();
756 - int* context_lens_ptr = context_lens.data_ptr<int>();
757 -
758 - constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
759 - int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
760 - int logits_size = padded_max_context_len * sizeof(float);
761 - int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
762 - int shared_mem_size = std::max(logits_size, outputs_size);
763 -
764 - dim3 grid(num_heads, num_seqs);
765 - dim3 block(NUM_THREADS);
766 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
767 - switch (head_size) {
768 - case 32:
769 - LAUNCH_MULTI_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
770 - break;
771 - case 64:
772 - LAUNCH_MULTI_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
773 - break;
774 - case 80:
775 - LAUNCH_MULTI_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
776 - break;
777 - case 96:
778 - LAUNCH_MULTI_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
779 - break;
780 - case 128:
781 - LAUNCH_MULTI_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
782 - break;
783 - case 160:
784 - LAUNCH_MULTI_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
785 - break;
786 - case 192:
787 - LAUNCH_MULTI_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
788 - break;
789 - case 256:
790 - LAUNCH_MULTI_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
791 - break;
792 - default:
793 - assert(false);
794 - break;
795 - }
796 -}
797 -
798 -void multi_query_cached_kv_attention(
799 - torch::Tensor& cu_query_lens,
800 - torch::Tensor& out,
801 - torch::Tensor& query,
802 - torch::Tensor& key_cache,
803 - torch::Tensor& value_cache,
804 - float scale,
805 - torch::Tensor& block_tables,
806 - torch::Tensor& context_lens,
807 - int block_size,
808 - int max_context_len) {
809 -
810 - torch::Tensor query_lens = cu_query_lens.to(torch::kCPU);
399 +// namespace cacheflow {
400 +
401 +// // Grid: (num_heads, num_query_tokens).
402 +// template<
403 +// typename scalar_t,
404 +// int HEAD_SIZE,
405 +// int BLOCK_SIZE,
406 +// int NUM_THREADS>
407 +// __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
408 +// scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
409 +// const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
410 +// const int seq_start_idx,
411 +// const int seq_len,
412 +// const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
413 +// const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
414 +// const float scale,
415 +// const int* __restrict__ block_table, // [num_seqs, max_num_blocks_per_seq]
416 +// const int context_len,
417 +// const int max_num_blocks_per_seq,
418 +// const int q_stride) {
419 +// constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
420 +// constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
421 +// const int thread_idx = threadIdx.x;
422 +// const int warp_idx = thread_idx / WARP_SIZE;
423 +// const int lane = thread_idx % WARP_SIZE;
424 +
425 +// const int head_idx = blockIdx.x;
426 +// const int num_heads = gridDim.x;
427 +// const int seq_idx = blockIdx.y;
428 +
429 +// // A vector type to store a part of a key or a query.
430 +// // The vector size is configured in such a way that the threads in a thread group
431 +// // fetch or comput 16 bytes at a time.
432 +// // For example, if the size of a thread group is 4 and the data type is half,
433 +// // then the vector size is 16 / (4 * sizeof(half)) == 2.
434 +// constexpr int VEC_SIZE = 16 / (THREAD_GROUP_SIZE * sizeof(scalar_t));
435 +// using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
436 +// using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
437 +
438 +// constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
439 +// constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
440 +
441 +// const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
442 +// const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
443 +
444 +// // Load the query to registers.
445 +// // Each thread in a thread group has a different part of the query.
446 +// // For example, if the the thread group size is 4, then the first thread in the group
447 +// // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
448 +// // th vectors of the query, and so on.
449 +// // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
450 +// const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
451 +// Q_vec q_vecs[NUM_VECS_PER_THREAD];
452 +// #pragma unroll
453 +// for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
454 +// const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
455 +// q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
456 +// }
457 +
458 +// // Memory planning.
459 +// extern __shared__ char shared_mem[];
460 +// // NOTE(woosuk): We use FP32 logits and accumulation.
461 +// float *logits = reinterpret_cast<float*>(shared_mem);
462 +// // Workspace for reduction.
463 +// __shared__ float red_smem[2 * NUM_WARPS];
464 +
465 +// // x == THREAD_GROUP_SIZE * VEC_SIZE
466 +// // Each thread group fetches x elements from the key at a time.
467 +// constexpr int x = 16 / sizeof(scalar_t);
468 +// float qk_max = -FLT_MAX;
469 +
470 +// const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
471 +// const int mask_boundary = context_len - seq_len + 1 + (seq_idx - seq_start_idx);
472 +
473 +// // Iterate over the key blocks.
474 +// // Each warp fetches a block of keys for each iteration.
475 +// // Each thread group in a warp fetches a key from the block, and computes
476 +// // dot product with the query.
477 +// for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
478 +// const int physical_block_number = block_table[block_idx];
479 +// const int physical_block_offset = thread_group_idx % BLOCK_SIZE;
480 +// const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
481 +
482 +// // Load a key to registers.
483 +// // Each thread in a thread group has a different part of the key.
484 +// // For example, if the the thread group size is 4, then the first thread in the group
485 +// // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
486 +// // vectors of the key, and so on.
487 +// K_vec k_vecs[NUM_VECS_PER_THREAD];
488 +// #pragma unroll
489 +// for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
490 +// const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
491 +// + head_idx * HEAD_SIZE * BLOCK_SIZE
492 +// + physical_block_offset * x;
493 +// const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
494 +// const int offset1 = (vec_idx * VEC_SIZE) / x;
495 +// const int offset2 = (vec_idx * VEC_SIZE) % x;
496 +// k_vecs[i] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
497 +// }
498 +
499 +// // Compute dot product.
500 +// // This includes a reduction across the threads in the same thread group.
501 +// const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
502 +// const bool mask = token_idx >= mask_boundary;
503 +
504 +// if (thread_group_offset == 0) {
505 +// // Store the partial reductions to shared memory.
506 +// // NOTE(woosuk): It is required to zero out the masked logits.
507 +// logits[token_idx] = mask ? 0.f : qk;
508 +// // Update the max value.
509 +// qk_max = mask ? qk_max : fmaxf(qk_max, qk);
510 +// }
511 +// }
512 +
513 +// // Perform reduction across the threads in the same warp to get the
514 +// // max qk value for each "warp" (not across the thread block yet).
515 +// // The 0-th thread of each thread group already has its max qk value.
516 +// #pragma unroll
517 +// for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
518 +// qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
519 +// }
520 +// if (lane == 0) {
521 +// red_smem[warp_idx] = qk_max;
522 +// }
523 +// __syncthreads();
524 +
525 +// // TODO(woosuk): Refactor this part.
526 +// // Get the max qk value for the sequence.
527 +// qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
528 +// #pragma unroll
529 +// for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
530 +// qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
531 +// }
532 +// // Broadcast the max qk value to all threads.
533 +// qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
534 +
535 +// // Get the sum of the exp values.
536 +// float exp_sum = 0.f;
537 +// for (int i = thread_idx; i < mask_boundary; i += NUM_THREADS) {
538 +// float val = __expf(logits[i] - qk_max);
539 +// logits[i] = val;
540 +// exp_sum += val;
541 +// }
542 +// exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
543 +
544 +// // Compute softmax.
545 +// const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
546 +// for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
547 +// logits[i] *= inv_sum;
548 +// }
549 +// __syncthreads();
550 +
551 +// // Each thread will fetch 16 bytes from the value cache at a time.
552 +// constexpr int V_VEC_SIZE = 16 / sizeof(scalar_t);
553 +// using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
554 +// using L_vec = typename FloatVec<V_vec>::Type;
555 +
556 +// constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
557 +// constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
558 +// constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
559 +
560 +// float accs[NUM_ROWS_PER_THREAD];
561 +// #pragma unroll
562 +// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
563 +// accs[i] = 0.f;
564 +// }
565 +
566 +// for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
567 +// const int physical_block_number = block_table[block_idx];
568 +// const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
569 +// const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
570 +// L_vec logits_vec = *reinterpret_cast<L_vec*>(logits + token_idx);
571 +
572 +// const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
573 +// + head_idx * HEAD_SIZE * BLOCK_SIZE;
574 +// #pragma unroll
575 +// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
576 +// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
577 +// if (row_idx < HEAD_SIZE) {
578 +// const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
579 +// V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
580 +// accs[i] += dot(logits_vec, cast_to_float(v_vec));
581 +// }
582 +// }
583 +// }
584 +
585 +// // Perform reduction within each warp.
586 +// #pragma unroll
587 +// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
588 +// float acc = accs[i];
589 +// #pragma unroll
590 +// for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
591 +// acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
592 +// }
593 +// accs[i] = acc;
594 +// }
595 +
596 +// // NOTE(woosuk): A barrier is required because the shared memory space for logits
597 +// // is reused for the output.
598 +// __syncthreads();
599 +
600 +// // Perform reduction across warps.
601 +// float* out_smem = reinterpret_cast<float*>(shared_mem);
602 +// #pragma unroll
603 +// for (int i = NUM_WARPS; i > 1; i /= 2) {
604 +// int mid = i / 2;
605 +// // Upper warps write to shared memory.
606 +// if (warp_idx >= mid && warp_idx < i) {
607 +// float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
608 +// #pragma unroll
609 +// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
610 +// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
611 +// if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
612 +// dst[row_idx] = accs[i];
613 +// }
614 +// }
615 +// }
616 +// __syncthreads();
617 +
618 +// // Lower warps update the output.
619 +// if (warp_idx < mid) {
620 +// const float* src = &out_smem[warp_idx * HEAD_SIZE];
621 +// #pragma unroll
622 +// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
623 +// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
624 +// if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
625 +// accs[i] += src[row_idx];
626 +// }
627 +// }
628 +// }
629 +// __syncthreads();
630 +// }
631 +
632 +// // Write the final output.
633 +// if (warp_idx == 0) {
634 +// scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
635 +// #pragma unroll
636 +// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
637 +// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
638 +// if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
639 +// convert_from_float(*(out_ptr + row_idx), accs[i]);
640 +// }
641 +// }
642 +// }
643 +// }
644 +
645 +
646 +// // Grid: (num_heads, num_query_tokens).
647 +// template<
648 +// typename scalar_t,
649 +// int HEAD_SIZE,
650 +// int BLOCK_SIZE,
651 +// int NUM_THREADS>
652 +// __global__ void multi_query_cached_kv_attention_kernel(
653 +// const int* cu_query_lens, // [num_prompts+1]
654 +// const int* seq_prompt_mapping, // [num_seqs] mapping from seq_idx to prompt_idx
655 +// scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
656 +// const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
657 +// const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
658 +// const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
659 +// const float scale,
660 +// const int* __restrict__ block_tables, // [num_prompts, max_num_blocks_per_seq]
661 +// const int* __restrict__ context_lens, // [num_prompts]
662 +// const int max_num_blocks_per_seq,
663 +// const int q_stride) {
664 +// const int seq_idx = blockIdx.y;
665 +// const int prompt_idx = seq_prompt_mapping[seq_idx];
666 +// const int seq_start_idx = cu_query_lens[prompt_idx];
667 +// const int seq_len = cu_query_lens[prompt_idx + 1] - seq_start_idx;
668 +// const int* block_table = block_tables + prompt_idx * max_num_blocks_per_seq;
669 +// const int context_len = context_lens[prompt_idx];
670 +// multi_query_cached_kv_attention_kernel_unoptimized_<
671 +// scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
672 +// out,
673 +// q,
674 +// seq_start_idx,
675 +// seq_len,
676 +// k_cache,
677 +// v_cache,
678 +// scale,
679 +// block_table,
680 +// context_len,
681 +// max_num_blocks_per_seq,
682 +// q_stride);
683 +// }
684 +
685 +// } // namespace cacheflow
686 +
687 +// #define LAUNCH_MULTI_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
688 +// cacheflow::multi_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
689 +// <<<grid, block, shared_mem_size, stream>>>( \
690 +// cu_query_lens_ptr, \
691 +// seq_prompt_mapping_ptr, \
692 +// out_ptr, \
693 +// query_ptr, \
694 +// key_cache_ptr, \
695 +// value_cache_ptr, \
696 +// scale, \
697 +// block_tables_ptr, \
698 +// context_lens_ptr, \
699 +// max_num_blocks_per_seq, \
700 +// query_stride);
701 +
702 +
703 +// // TODO(woosuk): Tune NUM_THREADS.
704 +// template<
705 +// typename T,
706 +// int BLOCK_SIZE,
707 +// int NUM_THREADS = 128>
708 +// void multi_query_cached_kv_attention_launcher(
709 +// torch::Tensor& cu_query_lens,
710 +// torch::Tensor& seq_prompt_mapping,
711 +// torch::Tensor& out,
712 +// torch::Tensor& query,
713 +// torch::Tensor& key_cache,
714 +// torch::Tensor& value_cache,
715 +// float scale,
716 +// torch::Tensor& block_tables,
717 +// torch::Tensor& context_lens,
718 +// int max_context_len) {
719 +// int num_seqs = query.size(0);
720 +// int num_heads = query.size(1);
721 +// int head_size = query.size(2);
722 +// int max_num_blocks_per_seq = block_tables.size(1);
723 +// int query_stride = query.stride(0);
724 +
725 +// int* cu_query_lens_ptr = cu_query_lens.data_ptr<int>();
726 +// int* seq_prompt_mapping_ptr = seq_prompt_mapping.data_ptr<int>();
727 +// T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
728 +// T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
729 +// T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
730 +// T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
731 +// int* block_tables_ptr = block_tables.data_ptr<int>();
732 +// int* context_lens_ptr = context_lens.data_ptr<int>();
733 +
734 +// constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
735 +// int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
736 +// int logits_size = padded_max_context_len * sizeof(float);
737 +// int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
738 +// int shared_mem_size = std::max(logits_size, outputs_size);
739 +
740 +// dim3 grid(num_heads, num_seqs);
741 +// dim3 block(NUM_THREADS);
742 +// const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
743 +// switch (head_size) {
744 +// case 32:
745 +// LAUNCH_MULTI_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
746 +// break;
747 +// case 64:
748 +// LAUNCH_MULTI_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
749 +// break;
750 +// case 80:
751 +// LAUNCH_MULTI_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
752 +// break;
753 +// case 96:
754 +// LAUNCH_MULTI_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
755 +// break;
756 +// case 128:
757 +// LAUNCH_MULTI_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
758 +// break;
759 +// case 160:
760 +// LAUNCH_MULTI_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
761 +// break;
762 +// case 192:
763 +// LAUNCH_MULTI_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
764 +// break;
765 +// case 256:
766 +// LAUNCH_MULTI_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
767 +// break;
768 +// default:
769 +// assert(false);
770 +// break;
771 +// }
772 +// }
773 +
774 +// void multi_query_cached_kv_attention(
775 +// torch::Tensor& cu_query_lens,
776 +// torch::Tensor& out,
777 +// torch::Tensor& query,
778 +// torch::Tensor& key_cache,
779 +// torch::Tensor& value_cache,
780 +// float scale,
781 +// torch::Tensor& block_tables,
782 +// torch::Tensor& context_lens,
783 +// int block_size,
784 +// int max_context_len) {
785 +
786 +// torch::Tensor query_lens = cu_query_lens.to(torch::kCPU);
787
812 - int num_queries = query_lens.size(0) - 1;
813 - const int* query_lens_ptr = query_lens.data_ptr<int>();
814 - int num_seqs = query.size(0);
815 -
816 - torch::Tensor cpu_tensor = torch::empty({num_seqs}, torch::dtype(torch::kInt32));
817 - auto accessor = cpu_tensor.accessor<int32_t, 1>();
818 - for (int i = 0, query_cursor = 0; i < num_seqs; ++i) {
819 - if (i >= query_lens_ptr[query_cursor + 1]) {
820 - ++query_cursor;
821 - }
822 - accessor[i] = query_cursor;
823 - }
824 -
825 - // TODO(suquark): This can be slow, as it to(torch::kCPU) and to(torch::kCUDA)
826 - // implicitly synchronizes the CPU and GPU. And we can avoid this issue by giving
827 - // the mapping as an input parameter. Let's do this optimization in a later PR.
828 - torch::Tensor seq_prompt_mapping = cpu_tensor.to(torch::kCUDA);
829 -
830 - // TODO(woosuk): Support BF16.
831 - if (query.element_size() == 2) {
832 - // Half.
833 - if (block_size == 8) {
834 - multi_query_cached_kv_attention_launcher<uint16_t, 8>(
835 - cu_query_lens,
836 - seq_prompt_mapping,
837 - out,
838 - query,
839 - key_cache,
840 - value_cache,
841 - scale,
842 - block_tables,
843 - context_lens,
844 - max_context_len);
845 - } else if (block_size == 16) {
846 - multi_query_cached_kv_attention_launcher<uint16_t, 16>(
847 - cu_query_lens,
848 - seq_prompt_mapping,
849 - out,
850 - query,
851 - key_cache,
852 - value_cache,
853 - scale,
854 - block_tables,
855 - context_lens,
856 - max_context_len);
857 - } else if (block_size == 32) {
858 - multi_query_cached_kv_attention_launcher<uint16_t, 32>(
859 - cu_query_lens,
860 - seq_prompt_mapping,
861 - out,
862 - query,
863 - key_cache,
864 - value_cache,
865 - scale,
866 - block_tables,
867 - context_lens,
868 - max_context_len);
869 - } else {
870 - assert(false);
871 - }
872 - } else if (query.element_size() == 4) {
873 - // Float.
874 - if (block_size == 8) {
875 - multi_query_cached_kv_attention_launcher<float, 8>(
876 - cu_query_lens,
877 - seq_prompt_mapping,
878 - out,
879 - query,
880 - key_cache,
881 - value_cache,
882 - scale,
883 - block_tables,
884 - context_lens,
885 - max_context_len);
886 - } else if (block_size == 16) {
887 - multi_query_cached_kv_attention_launcher<float, 16>(
888 - cu_query_lens,
889 - seq_prompt_mapping,
890 - out,
891 - query,
892 - key_cache,
893 - value_cache,
894 - scale,
895 - block_tables,
896 - context_lens,
897 - max_context_len);
898 - } else if (block_size == 32) {
899 - multi_query_cached_kv_attention_launcher<float, 32>(
900 - cu_query_lens,
901 - seq_prompt_mapping,
902 - out,
903 - query,
904 - key_cache,
905 - value_cache,
906 - scale,
907 - block_tables,
908 - context_lens,
909 - max_context_len);
910 - } else {
911 - assert(false);
912 - }
913 - } else {
914 - assert(false);
915 - }
916 -}
788 +// int num_queries = query_lens.size(0) - 1;
789 +// const int* query_lens_ptr = query_lens.data_ptr<int>();
790 +// int num_seqs = query.size(0);
791 +
792 +// torch::Tensor cpu_tensor = torch::empty({num_seqs}, torch::dtype(torch::kInt32));
793 +// auto accessor = cpu_tensor.accessor<int32_t, 1>();
794 +// for (int i = 0, query_cursor = 0; i < num_seqs; ++i) {
795 +// if (i >= query_lens_ptr[query_cursor + 1]) {
796 +// ++query_cursor;
797 +// }
798 +// accessor[i] = query_cursor;
799 +// }
800 +
801 +// // TODO(suquark): This can be slow, as it to(torch::kCPU) and to(torch::kCUDA)
802 +// // implicitly synchronizes the CPU and GPU. And we can avoid this issue by giving
803 +// // the mapping as an input parameter. Let's do this optimization in a later PR.
804 +// torch::Tensor seq_prompt_mapping = cpu_tensor.to(torch::kCUDA);
805 +
806 +// // TODO(woosuk): Support BF16.
807 +// if (query.element_size() == 2) {
808 +// // Half.
809 +// if (block_size == 8) {
810 +// multi_query_cached_kv_attention_launcher<uint16_t, 8>(
811 +// cu_query_lens,
812 +// seq_prompt_mapping,
813 +// out,
814 +// query,
815 +// key_cache,
816 +// value_cache,
817 +// scale,
818 +// block_tables,
819 +// context_lens,
820 +// max_context_len);
821 +// } else if (block_size == 16) {
822 +// multi_query_cached_kv_attention_launcher<uint16_t, 16>(
823 +// cu_query_lens,
824 +// seq_prompt_mapping,
825 +// out,
826 +// query,
827 +// key_cache,
828 +// value_cache,
829 +// scale,
830 +// block_tables,
831 +// context_lens,
832 +// max_context_len);
833 +// } else if (block_size == 32) {
834 +// multi_query_cached_kv_attention_launcher<uint16_t, 32>(
835 +// cu_query_lens,
836 +// seq_prompt_mapping,
837 +// out,
838 +// query,
839 +// key_cache,
840 +// value_cache,
841 +// scale,
842 +// block_tables,
843 +// context_lens,
844 +// max_context_len);
845 +// } else {
846 +// assert(false);
847 +// }
848 +// } else if (query.element_size() == 4) {
849 +// // Float.
850 +// if (block_size == 8) {
851 +// multi_query_cached_kv_attention_launcher<float, 8>(
852 +// cu_query_lens,
853 +// seq_prompt_mapping,
854 +// out,
855 +// query,
856 +// key_cache,
857 +// value_cache,
858 +// scale,
859 +// block_tables,
860 +// context_lens,
861 +// max_context_len);
862 +// } else if (block_size == 16) {
863 +// multi_query_cached_kv_attention_launcher<float, 16>(
864 +// cu_query_lens,
865 +// seq_prompt_mapping,
866 +// out,
867 +// query,
868 +// key_cache,
869 +// value_cache,
870 +// scale,
871 +// block_tables,
872 +// context_lens,
873 +// max_context_len);
874 +// } else if (block_size == 32) {
875 +// multi_query_cached_kv_attention_launcher<float, 32>(
876 +// cu_query_lens,
877 +// seq_prompt_mapping,
878 +// out,
879 +// query,
880 +// key_cache,
881 +// value_cache,
882 +// scale,
883 +// block_tables,
884 +// context_lens,
885 +// max_context_len);
886 +// } else {
887 +// assert(false);
888 +// }
889 +// } else {
890 +// assert(false);
891 +// }
892 +// }
893
894 #undef WARP_SIZE
895 +#undef MAX
896 +#undef MIN
csrc/cuda_primitives.h
+40 -18
@@ -1074,6 +1074,21 @@ inline __device__ float sum(Float8_ v)
1074
1075 ////////////////////////////////////////////////////////////////////////////////////////////////////
1076
1077 +inline __device__ float dot(float a, float b)
1078 +{
1079 + return a * b;
1080 +}
1081 +
1082 +////////////////////////////////////////////////////////////////////////////////////////////////////
1083 +
1084 +inline __device__ float dot(float2 a, float2 b)
1085 +{
1086 + float2 c = mul<float2, float2, float2>(a, b);
1087 + return c.x + c.y;
1088 +}
1089 +
1090 +////////////////////////////////////////////////////////////////////////////////////////////////////
1091 +
1092 inline __device__ float dot(Float4_ a, Float4_ b)
1093 {
1094 float2 acc = mul<float2, float2, float2>(a.x, b.x);
@@ -1253,37 +1268,44 @@ inline __device__ float convert_to_float(uint4 u)
1268
1269 ////////////////////////////////////////////////////////////////////////////////////////////////////
1270
1256 -inline __device__ float cast_to_float(float u)
1257 -{
1258 - return u;
1259 -}
1271 +// inline __device__ float cast_to_float(float u)
1272 +// {
1273 +// return u;
1274 +// }
1275
1276 ////////////////////////////////////////////////////////////////////////////////////////////////////
1277
1263 -inline __device__ float2 cast_to_float(float2 u)
1264 -{
1265 - return u;
1266 -}
1278 +// inline __device__ float2 cast_to_float(float2 u)
1279 +// {
1280 +// return u;
1281 +// }
1282
1283 ////////////////////////////////////////////////////////////////////////////////////////////////////
1284
1270 -inline __device__ float4 cast_to_float(float4 u)
1271 -{
1272 - return u;
1273 -}
1285 +// inline __device__ float4 cast_to_float(float4 u)
1286 +// {
1287 +// return u;
1288 +// }
1289
1290 ////////////////////////////////////////////////////////////////////////////////////////////////////
1291
1277 -inline __device__ Float4_ cast_to_float(Float4_ u)
1278 -{
1279 - return u;
1280 -}
1292 +// inline __device__ Float4_ cast_to_float(Float4_ u)
1293 +// {
1294 +// return u;
1295 +// }
1296 +
1297 +////////////////////////////////////////////////////////////////////////////////////////////////////
1298 +
1299 +// inline __device__ Float8_ cast_to_float(Float8_ u)
1300 +// {
1301 +// return u;
1302 +// }
1303
1304 ////////////////////////////////////////////////////////////////////////////////////////////////////
1305
1284 -inline __device__ Float8_ cast_to_float(Float8_ u)
1306 +inline __device__ float cast_to_float(uint16_t u)
1307 {
1286 - return u;
1308 + return half_to_float(u);
1309 }
1310
1311 ////////////////////////////////////////////////////////////////////////////////////////////////////