@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 21b3671bbc
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Basic attention kernel that supports cached KV + (multi-)prompts (#24)

Siyuan (Ryans) Zhuang committed Apr 4, 2023 at 20:34 UTC 21b3671bbc508662561ae95a418a26dbe71db356
3 files changed +622
csrc/attention.cpp
+16
@@ -11,9 +11,25 @@ void single_query_cached_kv_attention(
11 int block_size,
12 int max_context_len);
13
14 +void multi_query_cached_kv_attention(
15 + torch::Tensor& cu_query_lens,
16 + torch::Tensor& out,
17 + torch::Tensor& query,
18 + torch::Tensor& key_cache,
19 + torch::Tensor& value_cache,
20 + float scale,
21 + torch::Tensor& block_tables,
22 + torch::Tensor& context_lens,
23 + int block_size,
24 + int max_context_len);
25 +
26 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
27 m.def(
28 "single_query_cached_kv_attention",
29 &single_query_cached_kv_attention,
30 "Compute the attention between an input query and the cached key/value tensors");
31 + m.def(
32 + "multi_query_cached_kv_attention",
33 + &multi_query_cached_kv_attention,
34 + "Compute the attention between multiple input queries and the cached key/value tensors");
35 }
csrc/attention_kernels.cu
+463
@@ -254,6 +254,287 @@ __global__ void single_query_cached_kv_attention_kernel(
254 }
255 }
256
257 +
258 +// Grid: (num_heads, num_query_tokens).
259 +template<
260 + typename scalar_t,
261 + int HEAD_SIZE,
262 + int BLOCK_SIZE,
263 + int NUM_THREADS>
264 +__device__ void multi_query_cached_kv_attention_kernel_unoptimized_(
265 + scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
266 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
267 + const int seq_start_idx,
268 + const int seq_len,
269 + const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
270 + const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
271 + const float scale,
272 + const int* __restrict__ block_table, // [num_seqs, max_num_blocks_per_seq]
273 + const int context_len,
274 + const int max_num_blocks_per_seq) {
275 + constexpr int THREAD_GROUP_SIZE = WARP_SIZE / BLOCK_SIZE;
276 + constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
277 + const int thread_idx = threadIdx.x;
278 + const int warp_idx = thread_idx / WARP_SIZE;
279 + const int lane = thread_idx % WARP_SIZE;
280 +
281 + const int head_idx = blockIdx.x;
282 + const int num_heads = gridDim.x;
283 + const int seq_idx = blockIdx.y;
284 +
285 + // A vector type to store a part of a key or a query.
286 + // The vector size is configured in such a way that the threads in a thread group
287 + // fetch or comput 16 bytes at a time.
288 + // For example, if the size of a thread group is 4 and the data type is half,
289 + // then the vector size is 16 / (4 * sizeof(half)) == 2.
290 + constexpr int VEC_SIZE = 16 / (THREAD_GROUP_SIZE * sizeof(scalar_t));
291 + using K_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
292 + using Q_vec = typename Vec<scalar_t, VEC_SIZE>::Type;
293 +
294 + constexpr int NUM_ELEMS_PER_THREAD = HEAD_SIZE / THREAD_GROUP_SIZE;
295 + constexpr int NUM_VECS_PER_THREAD = NUM_ELEMS_PER_THREAD / VEC_SIZE;
296 +
297 + const int thread_group_idx = thread_idx / THREAD_GROUP_SIZE;
298 + const int thread_group_offset = thread_idx % THREAD_GROUP_SIZE;
299 +
300 + // Load the query to registers.
301 + // Each thread in a thread group has a different part of the query.
302 + // For example, if the the thread group size is 4, then the first thread in the group
303 + // has 0, 4, 8, ... th vectors of the query, and the second thread has 1, 5, 9, ...
304 + // th vectors of the query, and so on.
305 + const scalar_t* q_ptr = q + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
306 + Q_vec q_vecs[NUM_VECS_PER_THREAD];
307 +#pragma unroll
308 + for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
309 + const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
310 + q_vecs[i] = *reinterpret_cast<const Q_vec*>(q_ptr + vec_idx * VEC_SIZE);
311 + }
312 +
313 + // Memory planning.
314 + extern __shared__ char shared_mem[];
315 + // NOTE(woosuk): We use FP32 logits and accumulation.
316 + float *logits = reinterpret_cast<float*>(shared_mem);
317 + // Workspace for reduction.
318 + __shared__ float red_smem[2 * NUM_WARPS];
319 +
320 + // x == THREAD_GROUP_SIZE * VEC_SIZE
321 + // Each thread group fetches x elements from the key at a time.
322 + constexpr int x = 16 / sizeof(scalar_t);
323 + float qk_max = -FLT_MAX;
324 +
325 + const int num_blocks = (context_len + BLOCK_SIZE - 1) / BLOCK_SIZE;
326 + const int mask_boundary = context_len - seq_len + 1 + (seq_idx - seq_start_idx);
327 +
328 + // Iterate over the key blocks.
329 + // Each warp fetches a block of keys for each iteration.
330 + // Each thread group in a warp fetches a key from the block, and computes
331 + // dot product with the query.
332 + for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
333 + const int physical_block_number = block_table[block_idx];
334 + const int physical_block_offset = thread_group_idx % BLOCK_SIZE;
335 + const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
336 +
337 + // Load a key to registers.
338 + // Each thread in a thread group has a different part of the key.
339 + // For example, if the the thread group size is 4, then the first thread in the group
340 + // has 0, 4, 8, ... th vectors of the key, and the second thread has 1, 5, 9, ... th
341 + // vectors of the key, and so on.
342 + K_vec k_vecs[NUM_VECS_PER_THREAD];
343 +#pragma unroll
344 + for (int i = 0; i < NUM_VECS_PER_THREAD; i++) {
345 + const scalar_t* k_ptr = k_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
346 + + head_idx * HEAD_SIZE * BLOCK_SIZE
347 + + physical_block_offset * x;
348 + const int vec_idx = thread_group_offset + i * THREAD_GROUP_SIZE;
349 + const int offset1 = (vec_idx * VEC_SIZE) / x;
350 + const int offset2 = (vec_idx * VEC_SIZE) % x;
351 + k_vecs[i] = *reinterpret_cast<const K_vec*>(k_ptr + offset1 * BLOCK_SIZE * x + offset2);
352 + }
353 +
354 + // Compute dot product.
355 + // This includes a reduction across the threads in the same thread group.
356 + const float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs, k_vecs);
357 + const bool mask = token_idx >= mask_boundary;
358 +
359 + if (thread_group_offset == 0) {
360 + // Store the partial reductions to shared memory.
361 + // NOTE(woosuk): It is required to zero out the masked logits.
362 + logits[token_idx] = mask ? 0.f : qk;
363 + // Update the max value.
364 + qk_max = mask ? qk_max : fmaxf(qk_max, qk);
365 + }
366 + }
367 +
368 + // Perform reduction across the threads in the same warp to get the
369 + // max qk value for each "warp" (not across the thread block yet).
370 + // The 0-th thread of each thread group already has its max qk value.
371 +#pragma unroll
372 + for (int mask = WARP_SIZE / 2; mask >= THREAD_GROUP_SIZE; mask /= 2) {
373 + qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
374 + }
375 + if (lane == 0) {
376 + red_smem[warp_idx] = qk_max;
377 + }
378 + __syncthreads();
379 +
380 + // TODO(woosuk): Refactor this part.
381 + // Get the max qk value for the sequence.
382 + qk_max = lane < NUM_WARPS ? red_smem[lane] : -FLT_MAX;
383 +#pragma unroll
384 + for (int mask = NUM_WARPS / 2; mask >= 1; mask /= 2) {
385 + qk_max = fmaxf(qk_max, __shfl_xor_sync(uint32_t(-1), qk_max, mask));
386 + }
387 + // Broadcast the max qk value to all threads.
388 + qk_max = __shfl_sync(uint32_t(-1), qk_max, 0);
389 +
390 + // Get the sum of the exp values.
391 + float exp_sum = 0.f;
392 + for (int i = thread_idx; i < mask_boundary; i += NUM_THREADS) {
393 + float val = __expf(logits[i] - qk_max);
394 + logits[i] = val;
395 + exp_sum += val;
396 + }
397 + exp_sum = block_sum<NUM_WARPS>(&red_smem[NUM_WARPS], exp_sum);
398 +
399 + // Compute softmax.
400 + const float inv_sum = __fdividef(1.f, exp_sum + 1e-6f);
401 + for (int i = thread_idx; i < context_len; i += NUM_THREADS) {
402 + logits[i] *= inv_sum;
403 + }
404 + __syncthreads();
405 +
406 + // Each thread will fetch 16 bytes from the value cache at a time.
407 + constexpr int V_VEC_SIZE = 16 / sizeof(scalar_t);
408 + using V_vec = typename Vec<scalar_t, V_VEC_SIZE>::Type;
409 + using L_vec = typename FloatVec<V_vec>::Type;
410 +
411 + constexpr int NUM_V_VECS_PER_ROW = BLOCK_SIZE / V_VEC_SIZE;
412 + constexpr int NUM_ROWS_PER_ITER = WARP_SIZE / NUM_V_VECS_PER_ROW;
413 + constexpr int NUM_ROWS_PER_THREAD = (HEAD_SIZE + NUM_ROWS_PER_ITER - 1) / NUM_ROWS_PER_ITER;
414 +
415 + float accs[NUM_ROWS_PER_THREAD];
416 +#pragma unroll
417 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
418 + accs[i] = 0.f;
419 + }
420 +
421 + for (int block_idx = warp_idx; block_idx < num_blocks; block_idx += NUM_WARPS) {
422 + const int physical_block_number = block_table[block_idx];
423 + const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
424 + const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
425 + L_vec logits_vec = *reinterpret_cast<L_vec*>(logits + token_idx);
426 +
427 + const scalar_t* v_ptr = v_cache + physical_block_number * num_heads * HEAD_SIZE * BLOCK_SIZE
428 + + head_idx * HEAD_SIZE * BLOCK_SIZE;
429 +#pragma unroll
430 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
431 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
432 + if (row_idx < HEAD_SIZE) {
433 + const int offset = row_idx * BLOCK_SIZE + physical_block_offset;
434 + V_vec v_vec = *reinterpret_cast<const V_vec*>(v_ptr + offset);
435 + accs[i] += dot(logits_vec, cast_to_float(v_vec));
436 + }
437 + }
438 + }
439 +
440 + // Perform reduction within each warp.
441 +#pragma unroll
442 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
443 + float acc = accs[i];
444 +#pragma unroll
445 + for (int mask = NUM_V_VECS_PER_ROW / 2; mask >= 1; mask /= 2) {
446 + acc += __shfl_xor_sync(uint32_t(-1), acc, mask);
447 + }
448 + accs[i] = acc;
449 + }
450 +
451 + // NOTE(woosuk): A barrier is required because the shared memory space for logits
452 + // is reused for the output.
453 + __syncthreads();
454 +
455 + // Perform reduction across warps.
456 + float* out_smem = reinterpret_cast<float*>(shared_mem);
457 +#pragma unroll
458 + for (int i = NUM_WARPS; i > 1; i /= 2) {
459 + int mid = i / 2;
460 + // Upper warps write to shared memory.
461 + if (warp_idx >= mid && warp_idx < i) {
462 + float* dst = &out_smem[(warp_idx - mid) * HEAD_SIZE];
463 +#pragma unroll
464 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
465 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
466 + if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
467 + dst[row_idx] = accs[i];
468 + }
469 + }
470 + }
471 + __syncthreads();
472 +
473 + // Lower warps update the output.
474 + if (warp_idx < mid) {
475 + const float* src = &out_smem[warp_idx * HEAD_SIZE];
476 +#pragma unroll
477 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
478 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
479 + if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
480 + accs[i] += src[row_idx];
481 + }
482 + }
483 + }
484 + __syncthreads();
485 + }
486 +
487 + // Write the final output.
488 + if (warp_idx == 0) {
489 + scalar_t* out_ptr = out + seq_idx * num_heads * HEAD_SIZE + head_idx * HEAD_SIZE;
490 +#pragma unroll
491 + for (int i = 0; i < NUM_ROWS_PER_THREAD; i++) {
492 + const int row_idx = lane / NUM_V_VECS_PER_ROW + i * NUM_ROWS_PER_ITER;
493 + if (row_idx < HEAD_SIZE && lane % NUM_V_VECS_PER_ROW == 0) {
494 + convert_from_float(*(out_ptr + row_idx), accs[i]);
495 + }
496 + }
497 + }
498 +}
499 +
500 +
501 +// Grid: (num_heads, num_query_tokens).
502 +template<
503 + typename scalar_t,
504 + int HEAD_SIZE,
505 + int BLOCK_SIZE,
506 + int NUM_THREADS>
507 +__global__ void multi_query_cached_kv_attention_kernel(
508 + const int* cu_query_lens, // [num_prompts+1]
509 + const int* seq_prompt_mapping, // [num_seqs] mapping from seq_idx to prompt_idx
510 + scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
511 + const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
512 + const scalar_t* __restrict__ k_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
513 + const scalar_t* __restrict__ v_cache, // [num_blocks, num_heads, head_size, block_size]
514 + const float scale,
515 + const int* __restrict__ block_tables, // [num_prompts, max_num_blocks_per_seq]
516 + const int* __restrict__ context_lens, // [num_prompts]
517 + const int max_num_blocks_per_seq) {
518 + const int seq_idx = blockIdx.y;
519 + const int prompt_idx = seq_prompt_mapping[seq_idx];
520 + const int seq_start_idx = cu_query_lens[prompt_idx];
521 + const int seq_len = cu_query_lens[prompt_idx + 1] - seq_start_idx;
522 + const int* block_table = block_tables + prompt_idx * max_num_blocks_per_seq;
523 + const int context_len = context_lens[prompt_idx];
524 + multi_query_cached_kv_attention_kernel_unoptimized_<
525 + scalar_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS>(
526 + out,
527 + q,
528 + seq_start_idx,
529 + seq_len,
530 + k_cache,
531 + v_cache,
532 + scale,
533 + block_table,
534 + context_len,
535 + max_num_blocks_per_seq);
536 +}
537 +
538 } // namespace cacheflow
539
540 #define LAUNCH_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
@@ -402,4 +683,186 @@ void single_query_cached_kv_attention(
683 }
684 }
685
686 +
687 +#define LAUNCH_MULTI_ATTENTION_KERNEL(T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS) \
688 + cacheflow::multi_query_cached_kv_attention_kernel<T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS> \
689 + <<<grid, block, shared_mem_size, stream>>>( \
690 + cu_query_lens_ptr, \
691 + seq_prompt_mapping_ptr, \
692 + out_ptr, \
693 + query_ptr, \
694 + key_cache_ptr, \
695 + value_cache_ptr, \
696 + scale, \
697 + block_tables_ptr, \
698 + context_lens_ptr, \
699 + max_num_blocks_per_seq);
700 +
701 +
702 +// TODO(woosuk): Tune NUM_THREADS.
703 +template<
704 + typename T,
705 + int BLOCK_SIZE,
706 + int NUM_THREADS = 128>
707 +void multi_query_cached_kv_attention_launcher(
708 + torch::Tensor& cu_query_lens,
709 + torch::Tensor& seq_prompt_mapping,
710 + torch::Tensor& out,
711 + torch::Tensor& query,
712 + torch::Tensor& key_cache,
713 + torch::Tensor& value_cache,
714 + float scale,
715 + torch::Tensor& block_tables,
716 + torch::Tensor& context_lens,
717 + int max_context_len) {
718 + int num_seqs = query.size(0);
719 + int num_heads = query.size(1);
720 + int head_size = query.size(2);
721 + int max_num_blocks_per_seq = block_tables.size(1);
722 +
723 + int* cu_query_lens_ptr = cu_query_lens.data_ptr<int>();
724 + int* seq_prompt_mapping_ptr = seq_prompt_mapping.data_ptr<int>();
725 + T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
726 + T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
727 + T* key_cache_ptr = reinterpret_cast<T*>(key_cache.data_ptr());
728 + T* value_cache_ptr = reinterpret_cast<T*>(value_cache.data_ptr());
729 + int* block_tables_ptr = block_tables.data_ptr<int>();
730 + int* context_lens_ptr = context_lens.data_ptr<int>();
731 +
732 + constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
733 + int padded_max_context_len = ((max_context_len + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
734 + int logits_size = padded_max_context_len * sizeof(float);
735 + int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
736 + int shared_mem_size = std::max(logits_size, outputs_size);
737 +
738 + dim3 grid(num_heads, num_seqs);
739 + dim3 block(NUM_THREADS);
740 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
741 + switch (head_size) {
742 + case 32:
743 + LAUNCH_MULTI_ATTENTION_KERNEL(T, 32, BLOCK_SIZE, NUM_THREADS);
744 + break;
745 + case 64:
746 + LAUNCH_MULTI_ATTENTION_KERNEL(T, 64, BLOCK_SIZE, NUM_THREADS);
747 + break;
748 + case 80:
749 + LAUNCH_MULTI_ATTENTION_KERNEL(T, 80, BLOCK_SIZE, NUM_THREADS);
750 + break;
751 + case 96:
752 + LAUNCH_MULTI_ATTENTION_KERNEL(T, 96, BLOCK_SIZE, NUM_THREADS);
753 + break;
754 + case 128:
755 + LAUNCH_MULTI_ATTENTION_KERNEL(T, 128, BLOCK_SIZE, NUM_THREADS);
756 + break;
757 + case 160:
758 + LAUNCH_MULTI_ATTENTION_KERNEL(T, 160, BLOCK_SIZE, NUM_THREADS);
759 + break;
760 + case 192:
761 + LAUNCH_MULTI_ATTENTION_KERNEL(T, 192, BLOCK_SIZE, NUM_THREADS);
762 + break;
763 + case 256:
764 + LAUNCH_MULTI_ATTENTION_KERNEL(T, 256, BLOCK_SIZE, NUM_THREADS);
765 + break;
766 + default:
767 + assert(false);
768 + break;
769 + }
770 +}
771 +
772 +void multi_query_cached_kv_attention(
773 + torch::Tensor& cu_query_lens,
774 + torch::Tensor& out,
775 + torch::Tensor& query,
776 + torch::Tensor& key_cache,
777 + torch::Tensor& value_cache,
778 + float scale,
779 + torch::Tensor& block_tables,
780 + torch::Tensor& context_lens,
781 + int block_size,
782 + int max_context_len) {
783 +
784 + torch::Tensor query_lens = cu_query_lens.to(torch::kCPU);
785 +
786 + int num_queries = query_lens.size(0) - 1;
787 + const int* query_lens_ptr = query_lens.data_ptr<int>();
788 + int num_seqs = query.size(0);
789 +
790 + torch::Tensor cpu_tensor = torch::empty({num_seqs}, torch::dtype(torch::kInt32));
791 + auto accessor = cpu_tensor.accessor<int32_t, 1>();
792 + for (int i = 0, query_cursor = 0; i < num_seqs; ++i) {
793 + if (i >= query_lens_ptr[query_cursor + 1]) {
794 + ++query_cursor;
795 + }
796 + accessor[i] = query_cursor;
797 + }
798 +
799 + // TODO(suquark): This can be slow, as it to(torch::kCPU) and to(torch::kCUDA)
800 + // implicitly synchronizes the CPU and GPU. And we can avoid this issue by giving
801 + // the mapping as an input parameter. Let's do this optimization in a later PR.
802 + torch::Tensor seq_prompt_mapping = cpu_tensor.to(torch::kCUDA);
803 +
804 + // TODO(woosuk): Support BF16.
805 + if (query.element_size() == 2) {
806 + // Half.
807 + if (block_size == 8) {
808 + multi_query_cached_kv_attention_launcher<uint16_t, 8>(
809 + cu_query_lens,
810 + seq_prompt_mapping,
811 + out,
812 + query,
813 + key_cache,
814 + value_cache,
815 + scale,
816 + block_tables,
817 + context_lens,
818 + max_context_len);
819 + } else if (block_size == 16) {
820 + multi_query_cached_kv_attention_launcher<uint16_t, 16>(
821 + cu_query_lens,
822 + seq_prompt_mapping,
823 + out,
824 + query,
825 + key_cache,
826 + value_cache,
827 + scale,
828 + block_tables,
829 + context_lens,
830 + max_context_len);
831 + } else {
832 + assert(false);
833 + }
834 + } else if (query.element_size() == 4) {
835 + // Float.
836 + if (block_size == 8) {
837 + multi_query_cached_kv_attention_launcher<float, 8>(
838 + cu_query_lens,
839 + seq_prompt_mapping,
840 + out,
841 + query,
842 + key_cache,
843 + value_cache,
844 + scale,
845 + block_tables,
846 + context_lens,
847 + max_context_len);
848 + } else if (block_size == 16) {
849 + multi_query_cached_kv_attention_launcher<float, 16>(
850 + cu_query_lens,
851 + seq_prompt_mapping,
852 + out,
853 + query,
854 + key_cache,
855 + value_cache,
856 + scale,
857 + block_tables,
858 + context_lens,
859 + max_context_len);
860 + } else {
861 + assert(false);
862 + }
863 + } else {
864 + assert(false);
865 + }
866 +}
867 +
868 #undef WARP_SIZE
tests/kernels/attention.py
+143
@@ -97,6 +97,61 @@ def ref_multi_query_kv_attention(
97 return ref_output
98
99
100 +def ref_multi_query_cached_kv_attention(
101 + cu_query_lens: List[int],
102 + query: torch.Tensor,
103 + key_cache: torch.Tensor,
104 + value_cache: torch.Tensor,
105 + block_tables: torch.Tensor,
106 + context_lens: torch.Tensor,
107 + dtype: torch.dtype,
108 +) -> torch.Tensor:
109 + num_heads = value_cache.shape[1]
110 + head_size = value_cache.shape[2]
111 + block_size = value_cache.shape[3]
112 + scale = 1.0 / (head_size ** 0.5)
113 +
114 + num_queries = len(cu_query_lens) - 1
115 + ref_outputs = []
116 + for i in range(num_queries):
117 + start_idx = cu_query_lens[i]
118 + end_idx = cu_query_lens[i + 1]
119 + query_len = end_idx - start_idx
120 + context_len = int(context_lens[i])
121 + block_table = block_tables[i]
122 +
123 + # Create attention mask
124 + attn_mask = torch.triu(
125 + torch.ones(query_len, context_len), diagonal=context_len - query_len + 1) * -1e5
126 + attn_mask = attn_mask.to(dtype=dtype, device='cuda')
127 +
128 + keys = []
129 + values = []
130 + for j in range(context_len):
131 + block_number = int(block_table[j // block_size])
132 + block_offset = j % block_size
133 +
134 + k = key_cache[block_number, :, :, block_offset, :]
135 + k = k.reshape(num_heads, head_size)
136 + keys.append(k)
137 +
138 + v = value_cache[block_number, :, :, block_offset]
139 + values.append(v)
140 + keys = torch.stack(keys, dim=0)
141 + values = torch.stack(values, dim=0)
142 +
143 + ref_output = ref_masked_attention(
144 + query[start_idx:end_idx],
145 + keys,
146 + values,
147 + scale,
148 + attn_mask=attn_mask,
149 + )
150 + ref_outputs.append(ref_output)
151 + ref_output = torch.cat(ref_outputs, dim=0)
152 + return ref_output
153 +
154 +
155 def test_single_query_cached_kv_attention(
156 num_tokens: int,
157 num_heads: int,
@@ -216,6 +271,76 @@ def test_multi_query_kv_attention(
271 assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
272
273
274 +def test_multi_query_cached_kv_attention(
275 + num_queries: int,
276 + num_heads: int,
277 + head_size: int,
278 + block_size: int,
279 + num_blocks: int,
280 + dtype: torch.dtype,
281 +) -> None:
282 + query_lens = random.sample(range(1, MAX_SEQ_LEN), num_queries)
283 + cu_query_lens = [0]
284 + for query_len in query_lens:
285 + cu_query_lens.append(cu_query_lens[-1] + query_len)
286 + num_total_tokens = cu_query_lens[-1]
287 +
288 + query = torch.randn(
289 + num_total_tokens, num_heads, head_size, dtype=dtype, device='cuda')
290 + x = 16 // torch.tensor([], dtype=dtype).element_size()
291 + key_block_shape = (num_heads, head_size // x, block_size, x)
292 + key_cache = torch.randn(
293 + size=(num_blocks, *key_block_shape), dtype=dtype, device='cuda')
294 + value_block_shape = (num_heads, head_size, block_size)
295 + value_cache = torch.randn(
296 + size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
297 +
298 + cu_query_lens = torch.tensor(cu_query_lens, dtype=torch.int, device='cuda')
299 + context_lens = [
300 + query_len + random.randint(0, MAX_SEQ_LEN - query_len)
301 + for query_len in query_lens
302 + ]
303 + max_context_len = max(context_lens)
304 + context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
305 +
306 + max_num_blocks_per_seq = (max_context_len + block_size - 1) // block_size
307 + block_tables = []
308 + for _ in range(num_queries):
309 + block_table = [
310 + random.randint(0, num_blocks - 1)
311 + for _ in range(max_num_blocks_per_seq)
312 + ]
313 + block_tables.append(block_table)
314 + block_tables = torch.tensor(block_tables, dtype=torch.int, device='cuda')
315 +
316 + scale = float(1.0 / (head_size ** 0.5))
317 + output = torch.empty_like(query)
318 +
319 + attention_ops.multi_query_cached_kv_attention(
320 + cu_query_lens,
321 + output,
322 + query,
323 + key_cache,
324 + value_cache,
325 + scale,
326 + block_tables,
327 + context_lens,
328 + block_size,
329 + max_context_len,
330 + )
331 +
332 + ref_output = ref_multi_query_cached_kv_attention(
333 + cu_query_lens,
334 + query,
335 + key_cache,
336 + value_cache,
337 + block_tables,
338 + context_lens,
339 + dtype,
340 + )
341 + assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
342 +
343 +
344 @torch.inference_mode()
345 def test_attention(seed: int) -> None:
346 # NOTE(woosuk): Even when the seed is fixed, there is a chance that
@@ -237,6 +362,24 @@ def test_attention(seed: int) -> None:
362 dtype=dtype,
363 )
364
365 + # NOTE(siyuan): Same as above. Re-run the test if it fails. Also
366 + # note that the test is also more likely to fail due to the much
367 + # larger amount of tokens in the input may increase the variance.
368 + for dtype in [torch.half, torch.float]:
369 + for block_size in [8, 16]:
370 + for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
371 + print(f'Testing multi_query_cached_kv_attention with '
372 + f'dtype={dtype}, block_size={block_size}, '
373 + f'head_size={head_size}')
374 + test_multi_query_cached_kv_attention(
375 + num_queries=11,
376 + num_heads=3,
377 + head_size=head_size,
378 + block_size=block_size,
379 + num_blocks=1024,
380 + dtype=dtype,
381 + )
382 +
383 # NOTE(woosuk): FlashAttention does not support FP32.
384 for dtype in [torch.half]:
385 # NOTE(woosuk): FlashAttention does not support head_size > 128.