A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Basic attention kernel that supports cached KV + (multi-)prompts (#24)
Siyuan (Ryans) Zhuang committed
Apr 4, 2023 at 20:34 UTC
21b3671bbc508662561ae95a418a26dbe71db356
3 files changed
+622
csrc/attention.cpp
+16
@@ -11,9 +11,25 @@ void single_query_cached_kv_attention(
11
int block_size,
12
int max_context_len);
13
14
+void multi_query_cached_kv_attention(
15
+ torch::Tensor& cu_query_lens,
16
+ torch::Tensor& out,
17
+ torch::Tensor& query,
18
+ torch::Tensor& key_cache,
19
+ torch::Tensor& value_cache,
20
+ float scale,
21
+ torch::Tensor& block_tables,
22
+ torch::Tensor& context_lens,
23
+ int block_size,
24
+ int max_context_len);
25
+
26
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
27
m.def(
28
"single_query_cached_kv_attention",
29
&single_query_cached_kv_attention,
30
"Compute the attention between an input query and the cached key/value tensors");
31
+ m.def(
32
+ "multi_query_cached_kv_attention",
33
+ &multi_query_cached_kv_attention,
34
+ "Compute the attention between multiple input queries and the cached key/value tensors");
35
}
csrc/attention_kernels.cu
+463
@@ -254,6 +254,287 @@ __global__ void single_query_cached_kv_attention_kernel(
254
}
255
}
256
257
+
258
+// Grid: (num_heads, num_query_tokens).
259
+template<
260
+ typename scalar_t,
261
+ int HEAD_SIZE,
262
+ int BLOCK_SIZE,
263
+ int NUM_THREADS>
264
+__device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
265
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
266
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
267
+ const int seq_start_idx,
268
+ const int seq_len,
269
+ const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
270
+ const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
271
+ const float scale,
272
+ const int* __restrict__ block_table, // [num_seqs, max_num_blocks_per_seq]
273
+ const int context_len,
274
+ const int max_num_blocks_per_seq) {
275
+ constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
276
+ constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
277
+ const int thread_idx = threadIdx.x;
278
+ const int warp_idx = thread_idx / WARP_SIZE;
279
+ const int lane = thread_idx % WARP_SIZE;
280
+
281
+ const int head_idx = blockIdx.x;
282
+ const int num_heads = gridDim.x;
283
+ const int seq_idx = blockIdx.y;
284
+
285
+ // A vector type to store a part of a key or a query.
286
+ // The vector size is configured in such a way that the threads in a thread group
287
+ // fetch or comput 16 bytes at a time.
288
+ // For example, if the size of a thread group is 4 and the data type is half,
289
+ // then the vector size is 16 / (4 * sizeof(half)) == 2.
290
+ constexpr int VEC_SIZE = 16 / (THREAD_GROUP_SIZE * sizeof(scalar_t));
291
+ using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
292
+ using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
293
+
294
+ constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
295
+ constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
296
+
297
+ const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
298
+ const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
299
+
300
+ // Load the query to registers.
301
+ // Each thread in a thread group has a different part of the query.
302
+ // For example, if the the thread group size is 4, then the first thread in the group
303
+ // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
304
+ // th vectors of the query, and so on.
305
+ const scalar_t* q_ptr = q + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
306
+ Q_vec q_vecs[NUM_VECS_PER_THREAD];
307
+#pragma unroll
308
+ for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
309
+ const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
310
+ q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
311
+ }
312
+
313
+ // Memory planning.
314
+ extern __shared__ char shared_mem[];
315
+ // NOTE(woosuk): We use FP32 logits and accumulation.
316
+ float *logits = reinterpret_cast<float*>(shared_mem);
317
+ // Workspace for reduction.
318
+ __shared__ float red_smem[2 * NUM_WARPS];
319
+
320
+ // x == THREAD_GROUP_SIZE * VEC_SIZE
321
+ // Each thread group fetches x elements from the key at a time.
322
+ constexpr int x = 16 / sizeof(scalar_t);
323
+ float qk_max = -FLT_MAX;
324
+
325
+ const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
326
+ const int mask_boundary = context_len - seq_len + 1 + (seq_idx - seq_start_idx);
327
+
328
+ // Iterate over the key blocks.
329
+ // Each warp fetches a block of keys for each iteration.
330
+ // Each thread group in a warp fetches a key from the block, and computes
331
+ // dot product with the query.
332
+ for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
333
+ const int physical_block_number = block_table[block_idx];
334
+ const int physical_block_offset = thread_group_idx % BLOCK_SIZE;
335
+ const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
336
+
337
+ // Load a key to registers.
338
+ // Each thread in a thread group has a different part of the key.
339
+ // For example, if the the thread group size is 4, then the first thread in the group
340
+ // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
341
+ // vectors of the key, and so on.
342
+ K_vec k_vecs[NUM_VECS_PER_THREAD];
343
+#pragma unroll
344
+ for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
345
+ const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
346
+ + head_idx * HEAD_SIZE * BLOCK_SIZE
347
+ + physical_block_offset * x;
348
+ const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
349
+ const int offset1 = (vec_idx * VEC_SIZE) / x;
350
+ const int offset2 = (vec_idx * VEC_SIZE) % x;
351
+ k_vecs[i] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
352
+ }
353
+
354
+ // Compute dot product.
355
+ // This includes a reduction across the threads in the same thread group.
356
+ const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
357
+ const bool mask = token_idx >= mask_boundary;
358
+
359
+ if (thread_group_offset == 0) {
360
+ // Store the partial reductions to shared memory.
361
+ // NOTE(woosuk): It is required to zero out the masked logits.
362
+ logits[token_idx] = mask ? 0.f : qk;
363
+ // Update the max value.
364
+ qk_max = mask ? qk_max : fmaxf(qk_max, qk);
365
+ }
366
+ }
367
+
368
+ // Perform reduction across the threads in the same warp to get the
369
+ // max qk value for each "warp" (not across the thread block yet).
370
+ // The 0-th thread of each thread group already has its max qk value.
371
+#pragma unroll
372
+ for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
373
+ qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
374
+ }
375
+ if (lane == 0) {
376
+ red_smem[warp_idx] = qk_max;
377
+ }
378
+ __syncthreads();
379
+
380
+ // TODO(woosuk): Refactor this part.
381
+ // Get the max qk value for the sequence.
382
+ qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
383
+#pragma unroll
384
+ for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
385
+ qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
386
+ }
387
+ // Broadcast the max qk value to all threads.
388
+ qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
389
+
390
+ // Get the sum of the exp values.
391
+ float exp_sum = 0.f;
392
+ for (int i = thread_idx; i < mask_boundary; i += NUM_THREADS) {
393
+ float val = __expf(logits[i] - qk_max);
394
+ logits[i] = val;
395
+ exp_sum += val;
396
+ }
397
+ exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
398
+
399
+ // Compute softmax.
400
+ const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
401
+ for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
402
+ logits[i] *= inv_sum;
403
+ }
404
+ __syncthreads();
405
+
406
+ // Each thread will fetch 16 bytes from the value cache at a time.
407
+ constexpr int V_VEC_SIZE = 16 / sizeof(scalar_t);
408
+ using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
409
+ using L_vec = typename FloatVec<V_vec>::Type;
410
+
411
+ constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
412
+ constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
413
+ constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
414
+
415
+ float accs[NUM_ROWS_PER_THREAD];
416
+#pragma unroll
417
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
418
+ accs[i] = 0.f;
419
+ }
420
+
421
+ for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
422
+ const int physical_block_number = block_table[block_idx];
423
+ const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
424
+ const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
425
+ L_vec logits_vec = *reinterpret_cast<L_vec*>(logits + token_idx);
426
+
427
+ const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
428
+ + head_idx * HEAD_SIZE * BLOCK_SIZE;
429
+#pragma unroll
430
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
431
+ const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
432
+ if (row_idx < HEAD_SIZE) {
433
+ const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
434
+ V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
435
+ accs[i] += dot(logits_vec, cast_to_float(v_vec));
436
+ }
437
+ }
438
+ }
439
+
440
+ // Perform reduction within each warp.
441
+#pragma unroll
442
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
443
+ float acc = accs[i];
444
+#pragma unroll
445
+ for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
446
+ acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
447
+ }
448
+ accs[i] = acc;
449
+ }
450
+
451
+ // NOTE(woosuk): A barrier is required because the shared memory space for logits
452
+ // is reused for the output.
453
+ __syncthreads();
454
+
455
+ // Perform reduction across warps.
456
+ float* out_smem = reinterpret_cast<float*>(shared_mem);
457
+#pragma unroll
458
+ for (int i = NUM_WARPS; i > 1; i /= 2) {
459
+ int mid = i / 2;
460
+ // Upper warps write to shared memory.
461
+ if (warp_idx >= mid && warp_idx < i) {
462
+ float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
463
+#pragma unroll
464
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
465
+ const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
466
+ if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
467
+ dst[row_idx] = accs[i];
468
+ }
469
+ }
470
+ }
471
+ __syncthreads();
472
+
473
+ // Lower warps update the output.
474
+ if (warp_idx < mid) {
475
+ const float* src = &out_smem[warp_idx * HEAD_SIZE];
476
+#pragma unroll
477
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
478
+ const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
479
+ if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
480
+ accs[i] += src[row_idx];
481
+ }
482
+ }
483
+ }
484
+ __syncthreads();
485
+ }
486
+
487
+ // Write the final output.
488
+ if (warp_idx == 0) {
489
+ scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
490
+#pragma unroll
491
+ for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
492
+ const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
493
+ if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
494
+ convert_from_float(*(out_ptr + row_idx), accs[i]);
495
+ }
496
+ }
497
+ }
498
+}
499
+
500
+
501
+// Grid: (num_heads, num_query_tokens).
502
+template<
503
+ typename scalar_t,
504
+ int HEAD_SIZE,
505
+ int BLOCK_SIZE,
506
+ int NUM_THREADS>
507
+__global__ void multi_query_cached_kv_attention_kernel(
508
+ const int* cu_query_lens, // [num_prompts+1]
509
+ const int* seq_prompt_mapping, // [num_seqs] mapping from seq_idx to prompt_idx
510
+ scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
511
+ const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
512
+ const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
513
+ const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
514
+ const float scale,
515
+ const int* __restrict__ block_tables, // [num_prompts, max_num_blocks_per_seq]
516
+ const int* __restrict__ context_lens, // [num_prompts]
517
+ const int max_num_blocks_per_seq) {
518
+ const int seq_idx = blockIdx.y;
519
+ const int prompt_idx = seq_prompt_mapping[seq_idx];
520
+ const int seq_start_idx = cu_query_lens[prompt_idx];
521
+ const int seq_len = cu_query_lens[prompt_idx + 1] - seq_start_idx;
522
+ const int* block_table = block_tables + prompt_idx * max_num_blocks_per_seq;
523
+ const int context_len = context_lens[prompt_idx];
524
+ multi_query_cached_kv_attention_kernel_unoptimized_<
525
+ scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
526
+ out,
527
+ q,
528
+ seq_start_idx,
529
+ seq_len,
530
+ k_cache,
531
+ v_cache,
532
+ scale,
533
+ block_table,
534
+ context_len,
535
+ max_num_blocks_per_seq);
536
+}
537
+
538
} // namespace cacheflow
539
540
#define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
@@ -402,4 +683,186 @@ void single_query_cached_kv_attention(
683
}
684
}
685
686
+
687
+#define LAUNCH_MULTI_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
688
+ cacheflow::multi_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
689
+ <<<grid, block, shared_mem_size, stream>>>( \
690
+ cu_query_lens_ptr, \
691
+ seq_prompt_mapping_ptr, \
692
+ out_ptr, \
693
+ query_ptr, \
694
+ key_cache_ptr, \
695
+ value_cache_ptr, \
696
+ scale, \
697
+ block_tables_ptr, \
698
+ context_lens_ptr, \
699
+ max_num_blocks_per_seq);
700
+
701
+
702
+// TODO(woosuk): Tune NUM_THREADS.
703
+template<
704
+ typename T,
705
+ int BLOCK_SIZE,
706
+ int NUM_THREADS = 128>
707
+void multi_query_cached_kv_attention_launcher(
708
+ torch::Tensor& cu_query_lens,
709
+ torch::Tensor& seq_prompt_mapping,
710
+ torch::Tensor& out,
711
+ torch::Tensor& query,
712
+ torch::Tensor& key_cache,
713
+ torch::Tensor& value_cache,
714
+ float scale,
715
+ torch::Tensor& block_tables,
716
+ torch::Tensor& context_lens,
717
+ int max_context_len) {
718
+ int num_seqs = query.size(0);
719
+ int num_heads = query.size(1);
720
+ int head_size = query.size(2);
721
+ int max_num_blocks_per_seq = block_tables.size(1);
722
+
723
+ int* cu_query_lens_ptr = cu_query_lens.data_ptr<int>();
724
+ int* seq_prompt_mapping_ptr = seq_prompt_mapping.data_ptr<int>();
725
+ T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
726
+ T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
727
+ T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
728
+ T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
729
+ int* block_tables_ptr = block_tables.data_ptr<int>();
730
+ int* context_lens_ptr = context_lens.data_ptr<int>();
731
+
732
+ constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
733
+ int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
734
+ int logits_size = padded_max_context_len * sizeof(float);
735
+ int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
736
+ int shared_mem_size = std::max(logits_size, outputs_size);
737
+
738
+ dim3 grid(num_heads, num_seqs);
739
+ dim3 block(NUM_THREADS);
740
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
741
+ switch (head_size) {
742
+ case 32:
743
+ LAUNCH_MULTI_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
744
+ break;
745
+ case 64:
746
+ LAUNCH_MULTI_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
747
+ break;
748
+ case 80:
749
+ LAUNCH_MULTI_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
750
+ break;
751
+ case 96:
752
+ LAUNCH_MULTI_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
753
+ break;
754
+ case 128:
755
+ LAUNCH_MULTI_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
756
+ break;
757
+ case 160:
758
+ LAUNCH_MULTI_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
759
+ break;
760
+ case 192:
761
+ LAUNCH_MULTI_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
762
+ break;
763
+ case 256:
764
+ LAUNCH_MULTI_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
765
+ break;
766
+ default:
767
+ assert(false);
768
+ break;
769
+ }
770
+}
771
+
772
+void multi_query_cached_kv_attention(
773
+ torch::Tensor& cu_query_lens,
774
+ torch::Tensor& out,
775
+ torch::Tensor& query,
776
+ torch::Tensor& key_cache,
777
+ torch::Tensor& value_cache,
778
+ float scale,
779
+ torch::Tensor& block_tables,
780
+ torch::Tensor& context_lens,
781
+ int block_size,
782
+ int max_context_len) {
783
+
784
+ torch::Tensor query_lens = cu_query_lens.to(torch::kCPU);
785
+
786
+ int num_queries = query_lens.size(0) - 1;
787
+ const int* query_lens_ptr = query_lens.data_ptr<int>();
788
+ int num_seqs = query.size(0);
789
+
790
+ torch::Tensor cpu_tensor = torch::empty({num_seqs}, torch::dtype(torch::kInt32));
791
+ auto accessor = cpu_tensor.accessor<int32_t, 1>();
792
+ for (int i = 0, query_cursor = 0; i < num_seqs; ++i) {
793
+ if (i >= query_lens_ptr[query_cursor + 1]) {
794
+ ++query_cursor;
795
+ }
796
+ accessor[i] = query_cursor;
797
+ }
798
+
799
+ // TODO(suquark): This can be slow, as it to(torch::kCPU) and to(torch::kCUDA)
800
+ // implicitly synchronizes the CPU and GPU. And we can avoid this issue by giving
801
+ // the mapping as an input parameter. Let's do this optimization in a later PR.
802
+ torch::Tensor seq_prompt_mapping = cpu_tensor.to(torch::kCUDA);
803
+
804
+ // TODO(woosuk): Support BF16.
805
+ if (query.element_size() == 2) {
806
+ // Half.
807
+ if (block_size == 8) {
808
+ multi_query_cached_kv_attention_launcher<uint16_t, 8>(
809
+ cu_query_lens,
810
+ seq_prompt_mapping,
811
+ out,
812
+ query,
813
+ key_cache,
814
+ value_cache,
815
+ scale,
816
+ block_tables,
817
+ context_lens,
818
+ max_context_len);
819
+ } else if (block_size == 16) {
820
+ multi_query_cached_kv_attention_launcher<uint16_t, 16>(
821
+ cu_query_lens,
822
+ seq_prompt_mapping,
823
+ out,
824
+ query,
825
+ key_cache,
826
+ value_cache,
827
+ scale,
828
+ block_tables,
829
+ context_lens,
830
+ max_context_len);
831
+ } else {
832
+ assert(false);
833
+ }
834
+ } else if (query.element_size() == 4) {
835
+ // Float.
836
+ if (block_size == 8) {
837
+ multi_query_cached_kv_attention_launcher<float, 8>(
838
+ cu_query_lens,
839
+ seq_prompt_mapping,
840
+ out,
841
+ query,
842
+ key_cache,
843
+ value_cache,
844
+ scale,
845
+ block_tables,
846
+ context_lens,
847
+ max_context_len);
848
+ } else if (block_size == 16) {
849
+ multi_query_cached_kv_attention_launcher<float, 16>(
850
+ cu_query_lens,
851
+ seq_prompt_mapping,
852
+ out,
853
+ query,
854
+ key_cache,
855
+ value_cache,
856
+ scale,
857
+ block_tables,
858
+ context_lens,
859
+ max_context_len);
860
+ } else {
861
+ assert(false);
862
+ }
863
+ } else {
864
+ assert(false);
865
+ }
866
+}
867
+
868
#undef WARP_SIZE
tests/kernels/attention.py
+143
@@ -97,6 +97,61 @@ def ref_multi_query_kv_attention(
97
return ref_output
98
99
100
+def ref_multi_query_cached_kv_attention(
101
+ cu_query_lens: List[int],
102
+ query: torch.Tensor,
103
+ key_cache: torch.Tensor,
104
+ value_cache: torch.Tensor,
105
+ block_tables: torch.Tensor,
106
+ context_lens: torch.Tensor,
107
+ dtype: torch.dtype,
108
+) -> torch.Tensor:
109
+ num_heads = value_cache.shape[1]
110
+ head_size = value_cache.shape[2]
111
+ block_size = value_cache.shape[3]
112
+ scale = 1.0 / (head_size ** 0.5)
113
+
114
+ num_queries = len(cu_query_lens) - 1
115
+ ref_outputs = []
116
+ for i in range(num_queries):
117
+ start_idx = cu_query_lens[i]
118
+ end_idx = cu_query_lens[i + 1]
119
+ query_len = end_idx - start_idx
120
+ context_len = int(context_lens[i])
121
+ block_table = block_tables[i]
122
+
123
+ # Create attention mask
124
+ attn_mask = torch.triu(
125
+ torch.ones(query_len, context_len), diagonal=context_len - query_len + 1) * -1e5
126
+ attn_mask = attn_mask.to(dtype=dtype, device='cuda')
127
+
128
+ keys = []
129
+ values = []
130
+ for j in range(context_len):
131
+ block_number = int(block_table[j // block_size])
132
+ block_offset = j % block_size
133
+
134
+ k = key_cache[block_number, :, :, block_offset, :]
135
+ k = k.reshape(num_heads, head_size)
136
+ keys.append(k)
137
+
138
+ v = value_cache[block_number, :, :, block_offset]
139
+ values.append(v)
140
+ keys = torch.stack(keys, dim=0)
141
+ values = torch.stack(values, dim=0)
142
+
143
+ ref_output = ref_masked_attention(
144
+ query[start_idx:end_idx],
145
+ keys,
146
+ values,
147
+ scale,
148
+ attn_mask=attn_mask,
149
+ )
150
+ ref_outputs.append(ref_output)
151
+ ref_output = torch.cat(ref_outputs, dim=0)
152
+ return ref_output
153
+
154
+
155
def test_single_query_cached_kv_attention(
156
num_tokens: int,
157
num_heads: int,
@@ -216,6 +271,76 @@ def test_multi_query_kv_attention(
271
assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
272
273
274
+def test_multi_query_cached_kv_attention(
275
+ num_queries: int,
276
+ num_heads: int,
277
+ head_size: int,
278
+ block_size: int,
279
+ num_blocks: int,
280
+ dtype: torch.dtype,
281
+) -> None:
282
+ query_lens = random.sample(range(1, MAX_SEQ_LEN), num_queries)
283
+ cu_query_lens = [0]
284
+ for query_len in query_lens:
285
+ cu_query_lens.append(cu_query_lens[-1] + query_len)
286
+ num_total_tokens = cu_query_lens[-1]
287
+
288
+ query = torch.randn(
289
+ num_total_tokens, num_heads, head_size, dtype=dtype, device='cuda')
290
+ x = 16 // torch.tensor([], dtype=dtype).element_size()
291
+ key_block_shape = (num_heads, head_size // x, block_size, x)
292
+ key_cache = torch.randn(
293
+ size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
294
+ value_block_shape = (num_heads, head_size, block_size)
295
+ value_cache = torch.randn(
296
+ size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
297
+
298
+ cu_query_lens = torch.tensor(cu_query_lens, dtype=torch.int, device='cuda')
299
+ context_lens = [
300
+ query_len + random.randint(0, MAX_SEQ_LEN - query_len)
301
+ for query_len in query_lens
302
+ ]
303
+ max_context_len = max(context_lens)
304
+ context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
305
+
306
+ max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
307
+ block_tables = []
308
+ for _ in range(num_queries):
309
+ block_table = [
310
+ random.randint(0, num_blocks - 1)
311
+ for _ in range(max_num_blocks_per_seq)
312
+ ]
313
+ block_tables.append(block_table)
314
+ block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
315
+
316
+ scale = float(1.0 / (head_size ** 0.5))
317
+ output = torch.empty_like(query)
318
+
319
+ attention_ops.multi_query_cached_kv_attention(
320
+ cu_query_lens,
321
+ output,
322
+ query,
323
+ key_cache,
324
+ value_cache,
325
+ scale,
326
+ block_tables,
327
+ context_lens,
328
+ block_size,
329
+ max_context_len,
330
+ )
331
+
332
+ ref_output = ref_multi_query_cached_kv_attention(
333
+ cu_query_lens,
334
+ query,
335
+ key_cache,
336
+ value_cache,
337
+ block_tables,
338
+ context_lens,
339
+ dtype,
340
+ )
341
+ assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
342
+
343
+
344
@torch.inference_mode()
345
def test_attention(seed: int) -> None:
346
# NOTE(woosuk): Even when the seed is fixed, there is a chance that
@@ -237,6 +362,24 @@ def test_attention(seed: int) -> None:
362
dtype=dtype,
363
)
364
365
+ # NOTE(siyuan): Same as above. Re-run the test if it fails. Also
366
+ # note that the test is also more likely to fail due to the much
367
+ # larger amount of tokens in the input may increase the variance.
368
+ for dtype in [torch.half, torch.float]:
369
+ for block_size in [8, 16]:
370
+ for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
371
+ print(f'Testing multi_query_cached_kv_attention with '
372
+ f'dtype={dtype}, block_size={block_size}, '
373
+ f'head_size={head_size}')
374
+ test_multi_query_cached_kv_attention(
375
+ num_queries=11,
376
+ num_heads=3,
377
+ head_size=head_size,
378
+ block_size=block_size,
379
+ num_blocks=1024,
380
+ dtype=dtype,
381
+ )
382
+
383
# NOTE(woosuk): FlashAttention does not support FP32.
384
for dtype in [torch.half]:
385
# NOTE(woosuk): FlashAttention does not support head_size > 128.