A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Support various block sizes & Change default block size to 16 (#38)
Woosuk Kwon committed
Apr 15, 2023 at 09:03 UTC
0f4b32199ec6c5d16bc03767e36fff2d54559ff8
7 files changed
+602
-619
benchmark/benchmark_text_completion.py
+1
@@ -268,6 +268,7 @@ if __name__ == '__main__':
268
f'{model_name}-tp{args.tensor_parallel_size}',
269
sample_dir,
270
'cacheflow',
271
+ f'block{args.block_size}',
272
f'req-rate-{args.request_rate}',
273
f'seed{args.seed}',
274
f'duration-{args.duration}',
cacheflow/master/block_manager.py
-3
@@ -15,9 +15,6 @@ class BlockAllocator:
15
block_size: int,
16
num_blocks: int,
17
) -> None:
18
- if block_size not in [8, 16, 32]:
19
- raise ValueError(f'Unsupported block size: {block_size}'
20
- 'The block size must be one of {8, 16, 32}.')
18
self.device = device
19
self.block_size = block_size
20
self.num_blocks = num_blocks
cacheflow/master/scheduler.py
+2
-1
@@ -125,7 +125,8 @@ class Scheduler:
125
126
# Swap in the sequence groups in the SWAPPED state if possible.
127
self.swapped = self.policy.sort_by_priority(now, self.swapped)
128
- while self.swapped:
128
+ # FCFS
129
+ while self.swapped and not blocks_to_swap_out:
130
seq_group = self.swapped[0]
131
# If the sequence group has been preempted in this step, stop.
132
if seq_group in preempted:
cacheflow/master/server.py
+2
-2
@@ -180,9 +180,9 @@ def add_server_arguments(parser: argparse.ArgumentParser):
180
parser.add_argument('--pipeline-parallel-size', '-pp', type=int, default=1, help='number of pipeline stages')
181
parser.add_argument('--tensor-parallel-size', '-tp', type=int, default=1, help='number of tensor parallel replicas')
182
# KV cache arguments
183
- parser.add_argument('--block-size', type=int, default=8, choices=[8, 16, 32], help='token block size')
183
+ parser.add_argument('--block-size', type=int, default=16, choices=[1, 2, 4, 8, 16, 32, 64, 128, 256], help='token block size')
184
# NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
185
- parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
185
+ parser.add_argument('--dtype', type=str, default='half', choices=['half'], help='data type')
186
# TODO(woosuk): Support fine-grained seeds (e.g., seed per request).
187
parser.add_argument('--seed', type=int, default=0, help='random seed')
188
parser.add_argument('--swap-space', type=int, default=20, help='CPU swap space size (GiB) per GPU')
csrc/attention.cpp
-16
@@ -11,25 +11,9 @@ void single_query_cached_kv_attention(
11
int block_size,
12
int max_context_len);
13
14
-void multi_query_cached_kv_attention(
15
- torch::Tensor& cu_query_lens,
16
- torch::Tensor& out,
17
- torch::Tensor& query,
18
- torch::Tensor& key_cache,
19
- torch::Tensor& value_cache,
20
- float scale,
21
- torch::Tensor& block_tables,
22
- torch::Tensor& context_lens,
23
- int block_size,
24
- int max_context_len);
25
-
14
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
15
m.def(
16
"single_query_cached_kv_attention",
17
&single_query_cached_kv_attention,
18
"Compute the attention between an input query and the cached key/value tensors");
31
- m.def(
32
- "multi_query_cached_kv_attention",
33
- &multi_query_cached_kv_attention,
34
- "Compute the attention between multiple input queries and the cached key/value tensors");
19
}
csrc/attention_kernels.cu
+557
-579
@@ -8,6 +8,8 @@
8
#include <algorithm>
9
10
#define WARP_SIZE 32
11
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
12
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
13
14
namespace cacheflow {
15
@@ -27,7 +29,8 @@ __global__ void single_query_cached_kv_attention_kernel(
29
const int* __restrict__ context_lens, // [num_seqs]
30
const int max_num_blocks_per_seq,
31
const int q_stride) {
30
- constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
32
+ constexpr int THREAD_GROUP_SIZE = MAX(WARP_SIZE / BLOCK_SIZE, 1);
33
+ constexpr int NUM_TOKENS_PER_THREAD_GROUP = (BLOCK_SIZE + WARP_SIZE - 1) / WARP_SIZE;
34
constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
35
const int thread_idx = threadIdx.x;
36
const int warp_idx = thread_idx / WARP_SIZE;
@@ -39,10 +42,10 @@ __global__ void single_query_cached_kv_attention_kernel(
42
43
// A vector type to store a part of a key or a query.
44
// The vector size is configured in such a way that the threads in a thread group
42
- // fetch or comput 16 bytes at a time.
45
+ // fetch or compute 16 bytes at a time.
46
// For example, if the size of a thread group is 4 and the data type is half,
47
// then the vector size is 16 / (4 * sizeof(half)) == 2.
45
- constexpr int VEC_SIZE = 16 / (THREAD_GROUP_SIZE * sizeof(scalar_t));
48
+ constexpr int VEC_SIZE = MAX(16 / (THREAD_GROUP_SIZE * sizeof(scalar_t)), 1);
49
using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
50
using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
51
@@ -88,284 +91,42 @@ __global__ void single_query_cached_kv_attention_kernel(
91
// dot product with the query.
92
for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
93
const int physical_block_number = block_table[block_idx];
91
- const int physical_block_offset = thread_group_idx % BLOCK_SIZE;
92
- const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
94
95
// Load a key to registers.
96
// Each thread in a thread group has a different part of the key.
97
// For example, if the the thread group size is 4, then the first thread in the group
98
// has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
99
// vectors of the key, and so on.
99
- K_vec k_vecs[NUM_VECS_PER_THREAD];
100
-#pragma unroll
101
- for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
102
- const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
103
- + head_idx * HEAD_SIZE * BLOCK_SIZE
104
- + physical_block_offset * x;
105
- const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
106
- const int offset1 = (vec_idx * VEC_SIZE) / x;
107
- const int offset2 = (vec_idx * VEC_SIZE) % x;
108
- k_vecs[i] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
109
- }
110
-
111
- // Compute dot product.
112
- // This includes a reduction across the threads in the same thread group.
113
- const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
114
- const bool mask = token_idx >= context_len;
115
-
116
- if (thread_group_offset == 0) {
117
- // Store the partial reductions to shared memory.
118
- // NOTE(woosuk): It is required to zero out the masked logits.
119
- logits[token_idx] = mask ? 0.f : qk;
120
- // Update the max value.
121
- qk_max = mask ? qk_max : fmaxf(qk_max, qk);
122
- }
123
- }
124
-
125
- // Perform reduction across the threads in the same warp to get the
126
- // max qk value for each "warp" (not across the thread block yet).
127
- // The 0-th thread of each thread group already has its max qk value.
128
-#pragma unroll
129
- for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
130
- qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
131
- }
132
- if (lane == 0) {
133
- red_smem[warp_idx] = qk_max;
134
- }
135
- __syncthreads();
136
-
137
- // TODO(woosuk): Refactor this part.
138
- // Get the max qk value for the sequence.
139
- qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
140
-#pragma unroll
141
- for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
142
- qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
143
- }
144
- // Broadcast the max qk value to all threads.
145
- qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
146
-
147
- // Get the sum of the exp values.
148
- float exp_sum = 0.f;
149
- for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
150
- float val = __expf(logits[i] - qk_max);
151
- logits[i] = val;
152
- exp_sum += val;
153
- }
154
- exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
155
-
156
- // Compute softmax.
157
- const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
158
- for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
159
- logits[i] *= inv_sum;
160
- }
161
- __syncthreads();
162
-
163
- // Each thread will fetch 16 bytes from the value cache at a time.
164
- constexpr int V_VEC_SIZE = 16 / sizeof(scalar_t);
165
- using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
166
- using L_vec = typename FloatVec<V_vec>::Type;
167
-
168
- constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
169
- constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
170
- constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
171
-
172
- float accs[NUM_ROWS_PER_THREAD];
173
-#pragma unroll
174
- for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
175
- accs[i] = 0.f;
176
- }
177
-
178
- for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
179
- const int physical_block_number = block_table[block_idx];
180
- const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
181
- const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
182
- L_vec logits_vec = *reinterpret_cast<L_vec*>(logits + token_idx);
183
-
184
- const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
185
- + head_idx * HEAD_SIZE * BLOCK_SIZE;
186
-#pragma unroll
187
- for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
188
- const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
189
- if (row_idx < HEAD_SIZE) {
190
- const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
191
- V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
192
- accs[i] += dot(logits_vec, cast_to_float(v_vec));
193
- }
194
- }
195
- }
196
-
197
- // Perform reduction within each warp.
198
-#pragma unroll
199
- for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
200
- float acc = accs[i];
201
-#pragma unroll
202
- for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
203
- acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
204
- }
205
- accs[i] = acc;
206
- }
100
+ for (int i = 0; i < NUM_TOKENS_PER_THREAD_GROUP; i++) {
101
+ const int physical_block_offset = (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;
102
+ const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
103
+ K_vec k_vecs[NUM_VECS_PER_THREAD];
104
208
- // NOTE(woosuk): A barrier is required because the shared memory space for logits
209
- // is reused for the output.
210
- __syncthreads();
211
-
212
- // Perform reduction across warps.
213
- float* out_smem = reinterpret_cast<float*>(shared_mem);
214
-#pragma unroll
215
- for (int i = NUM_WARPS; i > 1; i /= 2) {
216
- int mid = i / 2;
217
- // Upper warps write to shared memory.
218
- if (warp_idx >= mid && warp_idx < i) {
219
- float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
105
#pragma unroll
221
- for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
222
- const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
223
- if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
224
- dst[row_idx] = accs[i];
225
- }
106
+ for (int j = 0; j < NUM_VECS_PER_THREAD; j++) {
107
+ const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
108
+ + head_idx * HEAD_SIZE * BLOCK_SIZE
109
+ + physical_block_offset * x;
110
+ const int vec_idx = thread_group_offset + j * THREAD_GROUP_SIZE;
111
+ const int offset1 = (vec_idx * VEC_SIZE) / x;
112
+ const int offset2 = (vec_idx * VEC_SIZE) % x;
113
+ k_vecs[j] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
114
}
227
- }
228
- __syncthreads();
115
230
- // Lower warps update the output.
231
- if (warp_idx < mid) {
232
- const float* src = &out_smem[warp_idx * HEAD_SIZE];
233
-#pragma unroll
234
- for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
235
- const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
236
- if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
237
- accs[i] += src[row_idx];
238
- }
239
- }
240
- }
241
- __syncthreads();
242
- }
243
-
244
- // Write the final output.
245
- if (warp_idx == 0) {
246
- scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
247
-#pragma unroll
248
- for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
249
- const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
250
- if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
251
- convert_from_float(*(out_ptr + row_idx), accs[i]);
116
+ // Compute dot product.
117
+ // This includes a reduction across the threads in the same thread group.
118
+ const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
119
+ const bool mask = token_idx >= context_len;
120
+
121
+ if (thread_group_offset == 0) {
122
+ // Store the partial reductions to shared memory.
123
+ // NOTE(woosuk): It is required to zero out the masked logits.
124
+ logits[token_idx] = mask ? 0.f : qk;
125
+ // Update the max value.
126
+ qk_max = mask ? qk_max : fmaxf(qk_max, qk);
127
}
128
}
129
}
255
-}
256
-
257
-
258
-// Grid: (num_heads, num_query_tokens).
259
-template<
260
- typename scalar_t,
261
- int HEAD_SIZE,
262
- int BLOCK_SIZE,
263
- int NUM_THREADS>
264
-__device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
265
- scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
266
- const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
267
- const int seq_start_idx,
268
- const int seq_len,
269
- const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
270
- const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
271
- const float scale,
272
- const int* __restrict__ block_table, // [num_seqs, max_num_blocks_per_seq]
273
- const int context_len,
274
- const int max_num_blocks_per_seq,
275
- const int q_stride) {
276
- constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
277
- constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
278
- const int thread_idx = threadIdx.x;
279
- const int warp_idx = thread_idx / WARP_SIZE;
280
- const int lane = thread_idx % WARP_SIZE;
281
-
282
- const int head_idx = blockIdx.x;
283
- const int num_heads = gridDim.x;
284
- const int seq_idx = blockIdx.y;
285
-
286
- // A vector type to store a part of a key or a query.
287
- // The vector size is configured in such a way that the threads in a thread group
288
- // fetch or comput 16 bytes at a time.
289
- // For example, if the size of a thread group is 4 and the data type is half,
290
- // then the vector size is 16 / (4 * sizeof(half)) == 2.
291
- constexpr int VEC_SIZE = 16 / (THREAD_GROUP_SIZE * sizeof(scalar_t));
292
- using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
293
- using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
294
-
295
- constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
296
- constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
297
-
298
- const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
299
- const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
300
-
301
- // Load the query to registers.
302
- // Each thread in a thread group has a different part of the query.
303
- // For example, if the the thread group size is 4, then the first thread in the group
304
- // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
305
- // th vectors of the query, and so on.
306
- // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
307
- const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
308
- Q_vec q_vecs[NUM_VECS_PER_THREAD];
309
-#pragma unroll
310
- for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
311
- const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
312
- q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
313
- }
314
-
315
- // Memory planning.
316
- extern __shared__ char shared_mem[];
317
- // NOTE(woosuk): We use FP32 logits and accumulation.
318
- float *logits = reinterpret_cast<float*>(shared_mem);
319
- // Workspace for reduction.
320
- __shared__ float red_smem[2 * NUM_WARPS];
321
-
322
- // x == THREAD_GROUP_SIZE * VEC_SIZE
323
- // Each thread group fetches x elements from the key at a time.
324
- constexpr int x = 16 / sizeof(scalar_t);
325
- float qk_max = -FLT_MAX;
326
-
327
- const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
328
- const int mask_boundary = context_len - seq_len + 1 + (seq_idx - seq_start_idx);
329
-
330
- // Iterate over the key blocks.
331
- // Each warp fetches a block of keys for each iteration.
332
- // Each thread group in a warp fetches a key from the block, and computes
333
- // dot product with the query.
334
- for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
335
- const int physical_block_number = block_table[block_idx];
336
- const int physical_block_offset = thread_group_idx % BLOCK_SIZE;
337
- const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
338
-
339
- // Load a key to registers.
340
- // Each thread in a thread group has a different part of the key.
341
- // For example, if the the thread group size is 4, then the first thread in the group
342
- // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
343
- // vectors of the key, and so on.
344
- K_vec k_vecs[NUM_VECS_PER_THREAD];
345
-#pragma unroll
346
- for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
347
- const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
348
- + head_idx * HEAD_SIZE * BLOCK_SIZE
349
- + physical_block_offset * x;
350
- const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
351
- const int offset1 = (vec_idx * VEC_SIZE) / x;
352
- const int offset2 = (vec_idx * VEC_SIZE) % x;
353
- k_vecs[i] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
354
- }
355
-
356
- // Compute dot product.
357
- // This includes a reduction across the threads in the same thread group.
358
- const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
359
- const bool mask = token_idx >= mask_boundary;
360
-
361
- if (thread_group_offset == 0) {
362
- // Store the partial reductions to shared memory.
363
- // NOTE(woosuk): It is required to zero out the masked logits.
364
- logits[token_idx] = mask ? 0.f : qk;
365
- // Update the max value.
366
- qk_max = mask ? qk_max : fmaxf(qk_max, qk);
367
- }
368
- }
130
131
// Perform reduction across the threads in the same warp to get the
132
// max qk value for each "warp" (not across the thread block yet).
@@ -391,7 +152,7 @@ __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
152
153
// Get the sum of the exp values.
154
float exp_sum = 0.f;
394
- for (int i = thread_idx; i < mask_boundary; i += NUM_THREADS) {
155
+ for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
156
float val = __expf(logits[i] - qk_max);
157
logits[i] = val;
158
exp_sum += val;
@@ -406,7 +167,7 @@ __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
167
__syncthreads();
168
169
// Each thread will fetch 16 bytes from the value cache at a time.
409
- constexpr int V_VEC_SIZE = 16 / sizeof(scalar_t);
170
+ constexpr int V_VEC_SIZE = MIN(16 / sizeof(scalar_t), BLOCK_SIZE);
171
using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
172
using L_vec = typename FloatVec<V_vec>::Type;
173
@@ -499,46 +260,6 @@ __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
260
}
261
}
262
502
-
503
-// Grid: (num_heads, num_query_tokens).
504
-template<
505
- typename scalar_t,
506
- int HEAD_SIZE,
507
- int BLOCK_SIZE,
508
- int NUM_THREADS>
509
-__global__ void multi_query_cached_kv_attention_kernel(
510
- const int* cu_query_lens, // [num_prompts+1]
511
- const int* seq_prompt_mapping, // [num_seqs] mapping from seq_idx to prompt_idx
512
- scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
513
- const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
514
- const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
515
- const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
516
- const float scale,
517
- const int* __restrict__ block_tables, // [num_prompts, max_num_blocks_per_seq]
518
- const int* __restrict__ context_lens, // [num_prompts]
519
- const int max_num_blocks_per_seq,
520
- const int q_stride) {
521
- const int seq_idx = blockIdx.y;
522
- const int prompt_idx = seq_prompt_mapping[seq_idx];
523
- const int seq_start_idx = cu_query_lens[prompt_idx];
524
- const int seq_len = cu_query_lens[prompt_idx + 1] - seq_start_idx;
525
- const int* block_table = block_tables + prompt_idx * max_num_blocks_per_seq;
526
- const int context_len = context_lens[prompt_idx];
527
- multi_query_cached_kv_attention_kernel_unoptimized_<
528
- scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
529
- out,
530
- q,
531
- seq_start_idx,
532
- seq_len,
533
- k_cache,
534
- v_cache,
535
- scale,
536
- block_table,
537
- context_len,
538
- max_num_blocks_per_seq,
539
- q_stride);
540
-}
541
-
263
} // namespace cacheflow
264
265
#define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
@@ -574,6 +295,9 @@ void single_query_cached_kv_attention_launcher(
295
int max_num_blocks_per_seq = block_tables.size(1);
296
int query_stride = query.stride(0);
297
298
+ int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
299
+ assert(head_size % thread_group_size == 0);
300
+
301
T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
302
T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
303
T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
@@ -621,6 +345,17 @@ void single_query_cached_kv_attention_launcher(
345
}
346
}
347
348
+#define CALL_KERNEL_LAUNCHER(T, BLOCK_SIZE) \
349
+ single_query_cached_kv_attention_launcher<T, BLOCK_SIZE>( \
350
+ out, \
351
+ query, \
352
+ key_cache, \
353
+ value_cache, \
354
+ scale, \
355
+ block_tables, \
356
+ context_lens, \
357
+ max_context_len);
358
+
359
void single_query_cached_kv_attention(
360
torch::Tensor& out, // [num_seqs, num_heads, head_size]
361
torch::Tensor& query, // [num_seqs, num_heads, head_size]
@@ -634,285 +369,528 @@ void single_query_cached_kv_attention(
369
// TODO(woosuk): Support BF16.
370
if (query.element_size() == 2) {
371
// Half.
637
- if (block_size == 8) {
638
- single_query_cached_kv_attention_launcher<uint16_t, 8>(
639
- out,
640
- query,
641
- key_cache,
642
- value_cache,
643
- scale,
644
- block_tables,
645
- context_lens,
646
- max_context_len);
647
- } else if (block_size == 16) {
648
- single_query_cached_kv_attention_launcher<uint16_t, 16>(
649
- out,
650
- query,
651
- key_cache,
652
- value_cache,
653
- scale,
654
- block_tables,
655
- context_lens,
656
- max_context_len);
657
- } else if (block_size == 32) {
658
- single_query_cached_kv_attention_launcher<uint16_t, 32>(
659
- out,
660
- query,
661
- key_cache,
662
- value_cache,
663
- scale,
664
- block_tables,
665
- context_lens,
666
- max_context_len);
667
- } else {
668
- assert(false);
669
- }
670
- } else if (query.element_size() == 4) {
671
- // Float.
672
- if (block_size == 8) {
673
- single_query_cached_kv_attention_launcher<float, 8>(
674
- out,
675
- query,
676
- key_cache,
677
- value_cache,
678
- scale,
679
- block_tables,
680
- context_lens,
681
- max_context_len);
372
+ if (block_size == 1) {
373
+ CALL_KERNEL_LAUNCHER(uint16_t, 1);
374
+ } else if (block_size == 2) {
375
+ CALL_KERNEL_LAUNCHER(uint16_t, 2);
376
+ } else if (block_size == 4) {
377
+ CALL_KERNEL_LAUNCHER(uint16_t, 4);
378
+ } else if (block_size == 8) {
379
+ CALL_KERNEL_LAUNCHER(uint16_t, 8);
380
} else if (block_size == 16) {
683
- single_query_cached_kv_attention_launcher<float, 16>(
684
- out,
685
- query,
686
- key_cache,
687
- value_cache,
688
- scale,
689
- block_tables,
690
- context_lens,
691
- max_context_len);
381
+ CALL_KERNEL_LAUNCHER(uint16_t, 16);
382
} else if (block_size == 32) {
693
- single_query_cached_kv_attention_launcher<float, 32>(
694
- out,
695
- query,
696
- key_cache,
697
- value_cache,
698
- scale,
699
- block_tables,
700
- context_lens,
701
- max_context_len);
383
+ CALL_KERNEL_LAUNCHER(uint16_t, 32);
384
+ } else if (block_size == 64) {
385
+ CALL_KERNEL_LAUNCHER(uint16_t, 64);
386
+ } else if (block_size == 128) {
387
+ CALL_KERNEL_LAUNCHER(uint16_t, 128);
388
+ } else if (block_size == 256) {
389
+ CALL_KERNEL_LAUNCHER(uint16_t, 256);
390
} else {
391
assert(false);
392
}
393
} else {
394
+ // Float.
395
assert(false);
396
}
397
}
398
710
-
711
-#define LAUNCH_MULTI_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
712
- cacheflow::multi_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
713
- <<<grid, block, shared_mem_size, stream>>>( \
714
- cu_query_lens_ptr, \
715
- seq_prompt_mapping_ptr, \
716
- out_ptr, \
717
- query_ptr, \
718
- key_cache_ptr, \
719
- value_cache_ptr, \
720
- scale, \
721
- block_tables_ptr, \
722
- context_lens_ptr, \
723
- max_num_blocks_per_seq, \
724
- query_stride);
725
-
726
-
727
-// TODO(woosuk): Tune NUM_THREADS.
728
-template<
729
- typename T,
730
- int BLOCK_SIZE,
731
- int NUM_THREADS = 128>
732
-void multi_query_cached_kv_attention_launcher(
733
- torch::Tensor& cu_query_lens,
734
- torch::Tensor& seq_prompt_mapping,
735
- torch::Tensor& out,
736
- torch::Tensor& query,
737
- torch::Tensor& key_cache,
738
- torch::Tensor& value_cache,
739
- float scale,
740
- torch::Tensor& block_tables,
741
- torch::Tensor& context_lens,
742
- int max_context_len) {
743
- int num_seqs = query.size(0);
744
- int num_heads = query.size(1);
745
- int head_size = query.size(2);
746
- int max_num_blocks_per_seq = block_tables.size(1);
747
- int query_stride = query.stride(0);
748
-
749
- int* cu_query_lens_ptr = cu_query_lens.data_ptr<int>();
750
- int* seq_prompt_mapping_ptr = seq_prompt_mapping.data_ptr<int>();
751
- T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
752
- T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
753
- T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
754
- T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
755
- int* block_tables_ptr = block_tables.data_ptr<int>();
756
- int* context_lens_ptr = context_lens.data_ptr<int>();
757
-
758
- constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
759
- int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
760
- int logits_size = padded_max_context_len * sizeof(float);
761
- int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
762
- int shared_mem_size = std::max(logits_size, outputs_size);
763
-
764
- dim3 grid(num_heads, num_seqs);
765
- dim3 block(NUM_THREADS);
766
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
767
- switch (head_size) {
768
- case 32:
769
- LAUNCH_MULTI_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
770
- break;
771
- case 64:
772
- LAUNCH_MULTI_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
773
- break;
774
- case 80:
775
- LAUNCH_MULTI_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
776
- break;
777
- case 96:
778
- LAUNCH_MULTI_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
779
- break;
780
- case 128:
781
- LAUNCH_MULTI_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
782
- break;
783
- case 160:
784
- LAUNCH_MULTI_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
785
- break;
786
- case 192:
787
- LAUNCH_MULTI_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
788
- break;
789
- case 256:
790
- LAUNCH_MULTI_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
791
- break;
792
- default:
793
- assert(false);
794
- break;
795
- }
796
-}
797
-
798
-void multi_query_cached_kv_attention(
799
- torch::Tensor& cu_query_lens,
800
- torch::Tensor& out,
801
- torch::Tensor& query,
802
- torch::Tensor& key_cache,
803
- torch::Tensor& value_cache,
804
- float scale,
805
- torch::Tensor& block_tables,
806
- torch::Tensor& context_lens,
807
- int block_size,
808
- int max_context_len) {
809
-
810
- torch::Tensor query_lens = cu_query_lens.to(torch::kCPU);
399
+// namespace cacheflow {
400
+
401
+// // Grid: (num_heads, num_query_tokens).
402
+// template<
403
+// typename scalar_t,
404
+// int HEAD_SIZE,
405
+// int BLOCK_SIZE,
406
+// int NUM_THREADS>
407
+// __device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
408
+// scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
409
+// const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
410
+// const int seq_start_idx,
411
+// const int seq_len,
412
+// const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
413
+// const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
414
+// const float scale,
415
+// const int* __restrict__ block_table, // [num_seqs, max_num_blocks_per_seq]
416
+// const int context_len,
417
+// const int max_num_blocks_per_seq,
418
+// const int q_stride) {
419
+// constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
420
+// constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
421
+// const int thread_idx = threadIdx.x;
422
+// const int warp_idx = thread_idx / WARP_SIZE;
423
+// const int lane = thread_idx % WARP_SIZE;
424
+
425
+// const int head_idx = blockIdx.x;
426
+// const int num_heads = gridDim.x;
427
+// const int seq_idx = blockIdx.y;
428
+
429
+// // A vector type to store a part of a key or a query.
430
+// // The vector size is configured in such a way that the threads in a thread group
431
+// // fetch or comput 16 bytes at a time.
432
+// // For example, if the size of a thread group is 4 and the data type is half,
433
+// // then the vector size is 16 / (4 * sizeof(half)) == 2.
434
+// constexpr int VEC_SIZE = 16 / (THREAD_GROUP_SIZE * sizeof(scalar_t));
435
+// using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
436
+// using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
437
+
438
+// constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
439
+// constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
440
+
441
+// const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
442
+// const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
443
+
444
+// // Load the query to registers.
445
+// // Each thread in a thread group has a different part of the query.
446
+// // For example, if the the thread group size is 4, then the first thread in the group
447
+// // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
448
+// // th vectors of the query, and so on.
449
+// // NOTE(woosuk): Because q is split from a qkv tensor, it may not be contiguous.
450
+// const scalar_t* q_ptr = q + seq_idx * q_stride + head_idx * HEAD_SIZE;
451
+// Q_vec q_vecs[NUM_VECS_PER_THREAD];
452
+// #pragma unroll
453
+// for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
454
+// const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
455
+// q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
456
+// }
457
+
458
+// // Memory planning.
459
+// extern __shared__ char shared_mem[];
460
+// // NOTE(woosuk): We use FP32 logits and accumulation.
461
+// float *logits = reinterpret_cast<float*>(shared_mem);
462
+// // Workspace for reduction.
463
+// __shared__ float red_smem[2 * NUM_WARPS];
464
+
465
+// // x == THREAD_GROUP_SIZE * VEC_SIZE
466
+// // Each thread group fetches x elements from the key at a time.
467
+// constexpr int x = 16 / sizeof(scalar_t);
468
+// float qk_max = -FLT_MAX;
469
+
470
+// const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
471
+// const int mask_boundary = context_len - seq_len + 1 + (seq_idx - seq_start_idx);
472
+
473
+// // Iterate over the key blocks.
474
+// // Each warp fetches a block of keys for each iteration.
475
+// // Each thread group in a warp fetches a key from the block, and computes
476
+// // dot product with the query.
477
+// for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
478
+// const int physical_block_number = block_table[block_idx];
479
+// const int physical_block_offset = thread_group_idx % BLOCK_SIZE;
480
+// const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
481
+
482
+// // Load a key to registers.
483
+// // Each thread in a thread group has a different part of the key.
484
+// // For example, if the the thread group size is 4, then the first thread in the group
485
+// // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
486
+// // vectors of the key, and so on.
487
+// K_vec k_vecs[NUM_VECS_PER_THREAD];
488
+// #pragma unroll
489
+// for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
490
+// const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
491
+// + head_idx * HEAD_SIZE * BLOCK_SIZE
492
+// + physical_block_offset * x;
493
+// const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
494
+// const int offset1 = (vec_idx * VEC_SIZE) / x;
495
+// const int offset2 = (vec_idx * VEC_SIZE) % x;
496
+// k_vecs[i] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
497
+// }
498
+
499
+// // Compute dot product.
500
+// // This includes a reduction across the threads in the same thread group.
501
+// const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
502
+// const bool mask = token_idx >= mask_boundary;
503
+
504
+// if (thread_group_offset == 0) {
505
+// // Store the partial reductions to shared memory.
506
+// // NOTE(woosuk): It is required to zero out the masked logits.
507
+// logits[token_idx] = mask ? 0.f : qk;
508
+// // Update the max value.
509
+// qk_max = mask ? qk_max : fmaxf(qk_max, qk);
510
+// }
511
+// }
512
+
513
+// // Perform reduction across the threads in the same warp to get the
514
+// // max qk value for each "warp" (not across the thread block yet).
515
+// // The 0-th thread of each thread group already has its max qk value.
516
+// #pragma unroll
517
+// for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
518
+// qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
519
+// }
520
+// if (lane == 0) {
521
+// red_smem[warp_idx] = qk_max;
522
+// }
523
+// __syncthreads();
524
+
525
+// // TODO(woosuk): Refactor this part.
526
+// // Get the max qk value for the sequence.
527
+// qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
528
+// #pragma unroll
529
+// for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
530
+// qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
531
+// }
532
+// // Broadcast the max qk value to all threads.
533
+// qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
534
+
535
+// // Get the sum of the exp values.
536
+// float exp_sum = 0.f;
537
+// for (int i = thread_idx; i < mask_boundary; i += NUM_THREADS) {
538
+// float val = __expf(logits[i] - qk_max);
539
+// logits[i] = val;
540
+// exp_sum += val;
541
+// }
542
+// exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
543
+
544
+// // Compute softmax.
545
+// const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
546
+// for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
547
+// logits[i] *= inv_sum;
548
+// }
549
+// __syncthreads();
550
+
551
+// // Each thread will fetch 16 bytes from the value cache at a time.
552
+// constexpr int V_VEC_SIZE = 16 / sizeof(scalar_t);
553
+// using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
554
+// using L_vec = typename FloatVec<V_vec>::Type;
555
+
556
+// constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
557
+// constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
558
+// constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
559
+
560
+// float accs[NUM_ROWS_PER_THREAD];
561
+// #pragma unroll
562
+// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
563
+// accs[i] = 0.f;
564
+// }
565
+
566
+// for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
567
+// const int physical_block_number = block_table[block_idx];
568
+// const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
569
+// const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
570
+// L_vec logits_vec = *reinterpret_cast<L_vec*>(logits + token_idx);
571
+
572
+// const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
573
+// + head_idx * HEAD_SIZE * BLOCK_SIZE;
574
+// #pragma unroll
575
+// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
576
+// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
577
+// if (row_idx < HEAD_SIZE) {
578
+// const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
579
+// V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
580
+// accs[i] += dot(logits_vec, cast_to_float(v_vec));
581
+// }
582
+// }
583
+// }
584
+
585
+// // Perform reduction within each warp.
586
+// #pragma unroll
587
+// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
588
+// float acc = accs[i];
589
+// #pragma unroll
590
+// for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
591
+// acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
592
+// }
593
+// accs[i] = acc;
594
+// }
595
+
596
+// // NOTE(woosuk): A barrier is required because the shared memory space for logits
597
+// // is reused for the output.
598
+// __syncthreads();
599
+
600
+// // Perform reduction across warps.
601
+// float* out_smem = reinterpret_cast<float*>(shared_mem);
602
+// #pragma unroll
603
+// for (int i = NUM_WARPS; i > 1; i /= 2) {
604
+// int mid = i / 2;
605
+// // Upper warps write to shared memory.
606
+// if (warp_idx >= mid && warp_idx < i) {
607
+// float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
608
+// #pragma unroll
609
+// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
610
+// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
611
+// if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
612
+// dst[row_idx] = accs[i];
613
+// }
614
+// }
615
+// }
616
+// __syncthreads();
617
+
618
+// // Lower warps update the output.
619
+// if (warp_idx < mid) {
620
+// const float* src = &out_smem[warp_idx * HEAD_SIZE];
621
+// #pragma unroll
622
+// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
623
+// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
624
+// if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
625
+// accs[i] += src[row_idx];
626
+// }
627
+// }
628
+// }
629
+// __syncthreads();
630
+// }
631
+
632
+// // Write the final output.
633
+// if (warp_idx == 0) {
634
+// scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
635
+// #pragma unroll
636
+// for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
637
+// const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
638
+// if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
639
+// convert_from_float(*(out_ptr + row_idx), accs[i]);
640
+// }
641
+// }
642
+// }
643
+// }
644
+
645
+
646
+// // Grid: (num_heads, num_query_tokens).
647
+// template<
648
+// typename scalar_t,
649
+// int HEAD_SIZE,
650
+// int BLOCK_SIZE,
651
+// int NUM_THREADS>
652
+// __global__ void multi_query_cached_kv_attention_kernel(
653
+// const int* cu_query_lens, // [num_prompts+1]
654
+// const int* seq_prompt_mapping, // [num_seqs] mapping from seq_idx to prompt_idx
655
+// scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
656
+// const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
657
+// const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
658
+// const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
659
+// const float scale,
660
+// const int* __restrict__ block_tables, // [num_prompts, max_num_blocks_per_seq]
661
+// const int* __restrict__ context_lens, // [num_prompts]
662
+// const int max_num_blocks_per_seq,
663
+// const int q_stride) {
664
+// const int seq_idx = blockIdx.y;
665
+// const int prompt_idx = seq_prompt_mapping[seq_idx];
666
+// const int seq_start_idx = cu_query_lens[prompt_idx];
667
+// const int seq_len = cu_query_lens[prompt_idx + 1] - seq_start_idx;
668
+// const int* block_table = block_tables + prompt_idx * max_num_blocks_per_seq;
669
+// const int context_len = context_lens[prompt_idx];
670
+// multi_query_cached_kv_attention_kernel_unoptimized_<
671
+// scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
672
+// out,
673
+// q,
674
+// seq_start_idx,
675
+// seq_len,
676
+// k_cache,
677
+// v_cache,
678
+// scale,
679
+// block_table,
680
+// context_len,
681
+// max_num_blocks_per_seq,
682
+// q_stride);
683
+// }
684
+
685
+// } // namespace cacheflow
686
+
687
+// #define LAUNCH_MULTI_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
688
+// cacheflow::multi_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
689
+// <<<grid, block, shared_mem_size, stream>>>( \
690
+// cu_query_lens_ptr, \
691
+// seq_prompt_mapping_ptr, \
692
+// out_ptr, \
693
+// query_ptr, \
694
+// key_cache_ptr, \
695
+// value_cache_ptr, \
696
+// scale, \
697
+// block_tables_ptr, \
698
+// context_lens_ptr, \
699
+// max_num_blocks_per_seq, \
700
+// query_stride);
701
+
702
+
703
+// // TODO(woosuk): Tune NUM_THREADS.
704
+// template<
705
+// typename T,
706
+// int BLOCK_SIZE,
707
+// int NUM_THREADS = 128>
708
+// void multi_query_cached_kv_attention_launcher(
709
+// torch::Tensor& cu_query_lens,
710
+// torch::Tensor& seq_prompt_mapping,
711
+// torch::Tensor& out,
712
+// torch::Tensor& query,
713
+// torch::Tensor& key_cache,
714
+// torch::Tensor& value_cache,
715
+// float scale,
716
+// torch::Tensor& block_tables,
717
+// torch::Tensor& context_lens,
718
+// int max_context_len) {
719
+// int num_seqs = query.size(0);
720
+// int num_heads = query.size(1);
721
+// int head_size = query.size(2);
722
+// int max_num_blocks_per_seq = block_tables.size(1);
723
+// int query_stride = query.stride(0);
724
+
725
+// int* cu_query_lens_ptr = cu_query_lens.data_ptr<int>();
726
+// int* seq_prompt_mapping_ptr = seq_prompt_mapping.data_ptr<int>();
727
+// T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
728
+// T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
729
+// T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
730
+// T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
731
+// int* block_tables_ptr = block_tables.data_ptr<int>();
732
+// int* context_lens_ptr = context_lens.data_ptr<int>();
733
+
734
+// constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
735
+// int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
736
+// int logits_size = padded_max_context_len * sizeof(float);
737
+// int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
738
+// int shared_mem_size = std::max(logits_size, outputs_size);
739
+
740
+// dim3 grid(num_heads, num_seqs);
741
+// dim3 block(NUM_THREADS);
742
+// const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
743
+// switch (head_size) {
744
+// case 32:
745
+// LAUNCH_MULTI_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
746
+// break;
747
+// case 64:
748
+// LAUNCH_MULTI_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
749
+// break;
750
+// case 80:
751
+// LAUNCH_MULTI_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
752
+// break;
753
+// case 96:
754
+// LAUNCH_MULTI_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
755
+// break;
756
+// case 128:
757
+// LAUNCH_MULTI_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
758
+// break;
759
+// case 160:
760
+// LAUNCH_MULTI_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
761
+// break;
762
+// case 192:
763
+// LAUNCH_MULTI_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
764
+// break;
765
+// case 256:
766
+// LAUNCH_MULTI_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
767
+// break;
768
+// default:
769
+// assert(false);
770
+// break;
771
+// }
772
+// }
773
+
774
+// void multi_query_cached_kv_attention(
775
+// torch::Tensor& cu_query_lens,
776
+// torch::Tensor& out,
777
+// torch::Tensor& query,
778
+// torch::Tensor& key_cache,
779
+// torch::Tensor& value_cache,
780
+// float scale,
781
+// torch::Tensor& block_tables,
782
+// torch::Tensor& context_lens,
783
+// int block_size,
784
+// int max_context_len) {
785
+
786
+// torch::Tensor query_lens = cu_query_lens.to(torch::kCPU);
787
812
- int num_queries = query_lens.size(0) - 1;
813
- const int* query_lens_ptr = query_lens.data_ptr<int>();
814
- int num_seqs = query.size(0);
815
-
816
- torch::Tensor cpu_tensor = torch::empty({num_seqs}, torch::dtype(torch::kInt32));
817
- auto accessor = cpu_tensor.accessor<int32_t, 1>();
818
- for (int i = 0, query_cursor = 0; i < num_seqs; ++i) {
819
- if (i >= query_lens_ptr[query_cursor + 1]) {
820
- ++query_cursor;
821
- }
822
- accessor[i] = query_cursor;
823
- }
824
-
825
- // TODO(suquark): This can be slow, as it to(torch::kCPU) and to(torch::kCUDA)
826
- // implicitly synchronizes the CPU and GPU. And we can avoid this issue by giving
827
- // the mapping as an input parameter. Let's do this optimization in a later PR.
828
- torch::Tensor seq_prompt_mapping = cpu_tensor.to(torch::kCUDA);
829
-
830
- // TODO(woosuk): Support BF16.
831
- if (query.element_size() == 2) {
832
- // Half.
833
- if (block_size == 8) {
834
- multi_query_cached_kv_attention_launcher<uint16_t, 8>(
835
- cu_query_lens,
836
- seq_prompt_mapping,
837
- out,
838
- query,
839
- key_cache,
840
- value_cache,
841
- scale,
842
- block_tables,
843
- context_lens,
844
- max_context_len);
845
- } else if (block_size == 16) {
846
- multi_query_cached_kv_attention_launcher<uint16_t, 16>(
847
- cu_query_lens,
848
- seq_prompt_mapping,
849
- out,
850
- query,
851
- key_cache,
852
- value_cache,
853
- scale,
854
- block_tables,
855
- context_lens,
856
- max_context_len);
857
- } else if (block_size == 32) {
858
- multi_query_cached_kv_attention_launcher<uint16_t, 32>(
859
- cu_query_lens,
860
- seq_prompt_mapping,
861
- out,
862
- query,
863
- key_cache,
864
- value_cache,
865
- scale,
866
- block_tables,
867
- context_lens,
868
- max_context_len);
869
- } else {
870
- assert(false);
871
- }
872
- } else if (query.element_size() == 4) {
873
- // Float.
874
- if (block_size == 8) {
875
- multi_query_cached_kv_attention_launcher<float, 8>(
876
- cu_query_lens,
877
- seq_prompt_mapping,
878
- out,
879
- query,
880
- key_cache,
881
- value_cache,
882
- scale,
883
- block_tables,
884
- context_lens,
885
- max_context_len);
886
- } else if (block_size == 16) {
887
- multi_query_cached_kv_attention_launcher<float, 16>(
888
- cu_query_lens,
889
- seq_prompt_mapping,
890
- out,
891
- query,
892
- key_cache,
893
- value_cache,
894
- scale,
895
- block_tables,
896
- context_lens,
897
- max_context_len);
898
- } else if (block_size == 32) {
899
- multi_query_cached_kv_attention_launcher<float, 32>(
900
- cu_query_lens,
901
- seq_prompt_mapping,
902
- out,
903
- query,
904
- key_cache,
905
- value_cache,
906
- scale,
907
- block_tables,
908
- context_lens,
909
- max_context_len);
910
- } else {
911
- assert(false);
912
- }
913
- } else {
914
- assert(false);
915
- }
916
-}
788
+// int num_queries = query_lens.size(0) - 1;
789
+// const int* query_lens_ptr = query_lens.data_ptr<int>();
790
+// int num_seqs = query.size(0);
791
+
792
+// torch::Tensor cpu_tensor = torch::empty({num_seqs}, torch::dtype(torch::kInt32));
793
+// auto accessor = cpu_tensor.accessor<int32_t, 1>();
794
+// for (int i = 0, query_cursor = 0; i < num_seqs; ++i) {
795
+// if (i >= query_lens_ptr[query_cursor + 1]) {
796
+// ++query_cursor;
797
+// }
798
+// accessor[i] = query_cursor;
799
+// }
800
+
801
+// // TODO(suquark): This can be slow, as it to(torch::kCPU) and to(torch::kCUDA)
802
+// // implicitly synchronizes the CPU and GPU. And we can avoid this issue by giving
803
+// // the mapping as an input parameter. Let's do this optimization in a later PR.
804
+// torch::Tensor seq_prompt_mapping = cpu_tensor.to(torch::kCUDA);
805
+
806
+// // TODO(woosuk): Support BF16.
807
+// if (query.element_size() == 2) {
808
+// // Half.
809
+// if (block_size == 8) {
810
+// multi_query_cached_kv_attention_launcher<uint16_t, 8>(
811
+// cu_query_lens,
812
+// seq_prompt_mapping,
813
+// out,
814
+// query,
815
+// key_cache,
816
+// value_cache,
817
+// scale,
818
+// block_tables,
819
+// context_lens,
820
+// max_context_len);
821
+// } else if (block_size == 16) {
822
+// multi_query_cached_kv_attention_launcher<uint16_t, 16>(
823
+// cu_query_lens,
824
+// seq_prompt_mapping,
825
+// out,
826
+// query,
827
+// key_cache,
828
+// value_cache,
829
+// scale,
830
+// block_tables,
831
+// context_lens,
832
+// max_context_len);
833
+// } else if (block_size == 32) {
834
+// multi_query_cached_kv_attention_launcher<uint16_t, 32>(
835
+// cu_query_lens,
836
+// seq_prompt_mapping,
837
+// out,
838
+// query,
839
+// key_cache,
840
+// value_cache,
841
+// scale,
842
+// block_tables,
843
+// context_lens,
844
+// max_context_len);
845
+// } else {
846
+// assert(false);
847
+// }
848
+// } else if (query.element_size() == 4) {
849
+// // Float.
850
+// if (block_size == 8) {
851
+// multi_query_cached_kv_attention_launcher<float, 8>(
852
+// cu_query_lens,
853
+// seq_prompt_mapping,
854
+// out,
855
+// query,
856
+// key_cache,
857
+// value_cache,
858
+// scale,
859
+// block_tables,
860
+// context_lens,
861
+// max_context_len);
862
+// } else if (block_size == 16) {
863
+// multi_query_cached_kv_attention_launcher<float, 16>(
864
+// cu_query_lens,
865
+// seq_prompt_mapping,
866
+// out,
867
+// query,
868
+// key_cache,
869
+// value_cache,
870
+// scale,
871
+// block_tables,
872
+// context_lens,
873
+// max_context_len);
874
+// } else if (block_size == 32) {
875
+// multi_query_cached_kv_attention_launcher<float, 32>(
876
+// cu_query_lens,
877
+// seq_prompt_mapping,
878
+// out,
879
+// query,
880
+// key_cache,
881
+// value_cache,
882
+// scale,
883
+// block_tables,
884
+// context_lens,
885
+// max_context_len);
886
+// } else {
887
+// assert(false);
888
+// }
889
+// } else {
890
+// assert(false);
891
+// }
892
+// }
893
894
#undef WARP_SIZE
895
+#undef MAX
896
+#undef MIN
csrc/cuda_primitives.h
+40
-18
@@ -1074,6 +1074,21 @@ inline __device__ float sum(Float8_ v)
1074
1075
////////////////////////////////////////////////////////////////////////////////////////////////////
1076
1077
+inline __device__ float dot(float a, float b)
1078
+{
1079
+ return a * b;
1080
+}
1081
+
1082
+////////////////////////////////////////////////////////////////////////////////////////////////////
1083
+
1084
+inline __device__ float dot(float2 a, float2 b)
1085
+{
1086
+ float2 c = mul<float2, float2, float2>(a, b);
1087
+ return c.x + c.y;
1088
+}
1089
+
1090
+////////////////////////////////////////////////////////////////////////////////////////////////////
1091
+
1092
inline __device__ float dot(Float4_ a, Float4_ b)
1093
{
1094
float2 acc = mul<float2, float2, float2>(a.x, b.x);
@@ -1253,37 +1268,44 @@ inline __device__ float convert_to_float(uint4 u)
1268
1269
////////////////////////////////////////////////////////////////////////////////////////////////////
1270
1256
-inline __device__ float cast_to_float(float u)
1257
-{
1258
- return u;
1259
-}
1271
+// inline __device__ float cast_to_float(float u)
1272
+// {
1273
+// return u;
1274
+// }
1275
1276
////////////////////////////////////////////////////////////////////////////////////////////////////
1277
1263
-inline __device__ float2 cast_to_float(float2 u)
1264
-{
1265
- return u;
1266
-}
1278
+// inline __device__ float2 cast_to_float(float2 u)
1279
+// {
1280
+// return u;
1281
+// }
1282
1283
////////////////////////////////////////////////////////////////////////////////////////////////////
1284
1270
-inline __device__ float4 cast_to_float(float4 u)
1271
-{
1272
- return u;
1273
-}
1285
+// inline __device__ float4 cast_to_float(float4 u)
1286
+// {
1287
+// return u;
1288
+// }
1289
1290
////////////////////////////////////////////////////////////////////////////////////////////////////
1291
1277
-inline __device__ Float4_ cast_to_float(Float4_ u)
1278
-{
1279
- return u;
1280
-}
1292
+// inline __device__ Float4_ cast_to_float(Float4_ u)
1293
+// {
1294
+// return u;
1295
+// }
1296
+
1297
+////////////////////////////////////////////////////////////////////////////////////////////////////
1298
+
1299
+// inline __device__ Float8_ cast_to_float(Float8_ u)
1300
+// {
1301
+// return u;
1302
+// }
1303
1304
////////////////////////////////////////////////////////////////////////////////////////////////////
1305
1284
-inline __device__ Float8_ cast_to_float(Float8_ u)
1306
+inline __device__ float cast_to_float(uint16_t u)
1307
{
1286
- return u;
1308
+ return half_to_float(u);
1309
}
1310
1311
////////////////////////////////////////////////////////////////////////////////////////////////////