@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 812c981fa0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

Splitting attention kernel file (#10091)

Signed-off-by: maleksan85 <maleksan@amd.com> Co-authored-by: Aleksandr Malyshev <maleksan@amd.com>

Aleksandr Malyshev committed Nov 11, 2024 at 22:55 UTC 812c981fa00a8b2b95865c6e76b6c3735a56d7d9
4 files changed +399 -328
CMakeLists.txt
+3 -2
@@ -37,7 +37,7 @@ set(PYTHON_SUPPORTED_VERSIONS "3.9" "3.10" "3.11" "3.12")
37 set(CUDA_SUPPORTED_ARCHS "7.0;7.5;8.0;8.6;8.9;9.0")
38
39 # Supported AMD GPU architectures.
40 -set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100")
40 +set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101")
41
42 #
43 # Supported/expected torch versions for CUDA/ROCm.
@@ -187,7 +187,8 @@ message(STATUS "FetchContent base directory: ${FETCHCONTENT_BASE_DIR}")
187
188 set(VLLM_EXT_SRC
189 "csrc/cache_kernels.cu"
190 - "csrc/attention/attention_kernels.cu"
190 + "csrc/attention/paged_attention_v1.cu"
191 + "csrc/attention/paged_attention_v2.cu"
192 "csrc/pos_encoding_kernels.cu"
193 "csrc/activation_kernels.cu"
194 "csrc/layernorm_kernels.cu"
csrc/attention/attention_kernels.cuh renamed
-326
@@ -670,332 +670,6 @@ __global__ void paged_attention_v2_reduce_kernel(
670
671 } // namespace vllm
672
673 -#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
674 - VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
675 - ((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, \
676 - BLOCK_SIZE, NUM_THREADS, \
677 - KV_DTYPE, IS_BLOCK_SPARSE>), \
678 - shared_mem_size); \
679 - vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
680 - NUM_THREADS, KV_DTYPE, IS_BLOCK_SPARSE> \
681 - <<<grid, block, shared_mem_size, stream>>>( \
682 - out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, \
683 - scale, block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
684 - alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
685 - k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
686 - blocksparse_vert_stride, blocksparse_block_size, \
687 - blocksparse_head_sliding_step);
688 -
689 -// TODO(woosuk): Tune NUM_THREADS.
690 -template <typename T, typename CACHE_T, int BLOCK_SIZE,
691 - vllm::Fp8KVCacheDataType KV_DTYPE, bool IS_BLOCK_SPARSE,
692 - int NUM_THREADS = 128>
693 -void paged_attention_v1_launcher(
694 - torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
695 - torch::Tensor& value_cache, int num_kv_heads, float scale,
696 - torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
697 - const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
698 - float v_scale, const int tp_rank, const int blocksparse_local_blocks,
699 - const int blocksparse_vert_stride, const int blocksparse_block_size,
700 - const int blocksparse_head_sliding_step) {
701 - int num_seqs = query.size(0);
702 - int num_heads = query.size(1);
703 - int head_size = query.size(2);
704 - int max_num_blocks_per_seq = block_tables.size(1);
705 - int q_stride = query.stride(0);
706 - int kv_block_stride = key_cache.stride(0);
707 - int kv_head_stride = key_cache.stride(1);
708 -
709 - [[maybe_unused]] int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
710 - assert(head_size % thread_group_size == 0);
711 -
712 - // NOTE: alibi_slopes is optional.
713 - const float* alibi_slopes_ptr =
714 - alibi_slopes
715 - ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
716 - : nullptr;
717 -
718 - T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
719 - T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
720 - CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
721 - CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
722 - int* block_tables_ptr = block_tables.data_ptr<int>();
723 - int* seq_lens_ptr = seq_lens.data_ptr<int>();
724 -
725 - constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
726 - int padded_max_seq_len =
727 - DIVIDE_ROUND_UP(max_seq_len, BLOCK_SIZE) * BLOCK_SIZE;
728 - int logits_size = padded_max_seq_len * sizeof(float);
729 - int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
730 - // Python-side check in vllm.worker.worker._check_if_can_support_max_seq_len
731 - // Keep that in sync with the logic here!
732 - int shared_mem_size = std::max(logits_size, outputs_size);
733 -
734 - dim3 grid(num_heads, num_seqs, 1);
735 - dim3 block(NUM_THREADS);
736 - const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
737 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
738 - switch (head_size) {
739 - // NOTE(woosuk): To reduce the compilation time, we only compile for the
740 - // head sizes that we use in the model. However, we can easily extend this
741 - // to support any head size which is a multiple of 16.
742 - case 64:
743 - LAUNCH_PAGED_ATTENTION_V1(64);
744 - break;
745 - case 80:
746 - LAUNCH_PAGED_ATTENTION_V1(80);
747 - break;
748 - case 96:
749 - LAUNCH_PAGED_ATTENTION_V1(96);
750 - break;
751 - case 112:
752 - LAUNCH_PAGED_ATTENTION_V1(112);
753 - break;
754 - case 120:
755 - LAUNCH_PAGED_ATTENTION_V1(120);
756 - break;
757 - case 128:
758 - LAUNCH_PAGED_ATTENTION_V1(128);
759 - break;
760 - case 192:
761 - LAUNCH_PAGED_ATTENTION_V1(192);
762 - break;
763 - case 256:
764 - LAUNCH_PAGED_ATTENTION_V1(256);
765 - break;
766 - default:
767 - TORCH_CHECK(false, "Unsupported head size: ", head_size);
768 - break;
769 - }
770 -}
771 -
772 -#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE, IS_BLOCK_SPARSE) \
773 - paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
774 - IS_BLOCK_SPARSE>( \
775 - out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
776 - seq_lens, max_seq_len, alibi_slopes, k_scale, v_scale, tp_rank, \
777 - blocksparse_local_blocks, blocksparse_vert_stride, \
778 - blocksparse_block_size, blocksparse_head_sliding_step);
779 -
780 -#define CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
781 - switch (is_block_sparse) { \
782 - case true: \
783 - CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, true); \
784 - break; \
785 - case false: \
786 - CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, false); \
787 - break; \
788 - }
789 -
790 -// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
791 -// 1, 2, 4, 64, 128, 256.
792 -#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
793 - switch (block_size) { \
794 - case 8: \
795 - CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 8, KV_DTYPE); \
796 - break; \
797 - case 16: \
798 - CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 16, KV_DTYPE); \
799 - break; \
800 - case 32: \
801 - CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 32, KV_DTYPE); \
802 - break; \
803 - default: \
804 - TORCH_CHECK(false, "Unsupported block size: ", block_size); \
805 - break; \
806 - }
807 -
808 -void paged_attention_v1(
809 - torch::Tensor& out, // [num_seqs, num_heads, head_size]
810 - torch::Tensor& query, // [num_seqs, num_heads, head_size]
811 - torch::Tensor&
812 - key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
813 - torch::Tensor&
814 - value_cache, // [num_blocks, num_heads, head_size, block_size]
815 - int64_t num_kv_heads, // [num_heads]
816 - double scale,
817 - torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
818 - torch::Tensor& seq_lens, // [num_seqs]
819 - int64_t block_size, int64_t max_seq_len,
820 - const c10::optional<torch::Tensor>& alibi_slopes,
821 - const std::string& kv_cache_dtype, double k_scale, double v_scale,
822 - const int64_t tp_rank, const int64_t blocksparse_local_blocks,
823 - const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
824 - const int64_t blocksparse_head_sliding_step) {
825 - const bool is_block_sparse = (blocksparse_vert_stride > 1);
826 -
827 - DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
828 - CALL_V1_LAUNCHER_BLOCK_SIZE)
829 -}
830 -
831 -#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
832 - vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
833 - NUM_THREADS, KV_DTYPE, IS_BLOCK_SPARSE, \
834 - PARTITION_SIZE> \
835 - <<<grid, block, shared_mem_size, stream>>>( \
836 - exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, key_cache_ptr, \
837 - value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
838 - seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
839 - kv_block_stride, kv_head_stride, k_scale, v_scale, tp_rank, \
840 - blocksparse_local_blocks, blocksparse_vert_stride, \
841 - blocksparse_block_size, blocksparse_head_sliding_step); \
842 - vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, \
843 - PARTITION_SIZE> \
844 - <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
845 - out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, seq_lens_ptr, \
846 - max_num_partitions);
847 -
848 -template <typename T, typename CACHE_T, int BLOCK_SIZE,
849 - vllm::Fp8KVCacheDataType KV_DTYPE, bool IS_BLOCK_SPARSE,
850 - int NUM_THREADS = 128, int PARTITION_SIZE = 512>
851 -void paged_attention_v2_launcher(
852 - torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
853 - torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
854 - torch::Tensor& value_cache, int num_kv_heads, float scale,
855 - torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
856 - const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
857 - float v_scale, const int tp_rank, const int blocksparse_local_blocks,
858 - const int blocksparse_vert_stride, const int blocksparse_block_size,
859 - const int blocksparse_head_sliding_step) {
860 - int num_seqs = query.size(0);
861 - int num_heads = query.size(1);
862 - int head_size = query.size(2);
863 - int max_num_blocks_per_seq = block_tables.size(1);
864 - int q_stride = query.stride(0);
865 - int kv_block_stride = key_cache.stride(0);
866 - int kv_head_stride = key_cache.stride(1);
867 -
868 - [[maybe_unused]] int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
869 - assert(head_size % thread_group_size == 0);
870 -
871 - // NOTE: alibi_slopes is optional.
872 - const float* alibi_slopes_ptr =
873 - alibi_slopes
874 - ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
875 - : nullptr;
876 -
877 - T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
878 - float* exp_sums_ptr = reinterpret_cast<float*>(exp_sums.data_ptr());
879 - float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
880 - T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
881 - T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
882 - CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
883 - CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
884 - int* block_tables_ptr = block_tables.data_ptr<int>();
885 - int* seq_lens_ptr = seq_lens.data_ptr<int>();
886 -
887 - constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
888 - int max_num_partitions = DIVIDE_ROUND_UP(max_seq_len, PARTITION_SIZE);
889 - int logits_size = PARTITION_SIZE * sizeof(float);
890 - int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
891 -
892 - // For paged attention v2 kernel.
893 - dim3 grid(num_heads, num_seqs, max_num_partitions);
894 - int shared_mem_size = std::max(logits_size, outputs_size);
895 - // For paged attention v2 reduce kernel.
896 - dim3 reduce_grid(num_heads, num_seqs);
897 - int reduce_shared_mem_size = 2 * max_num_partitions * sizeof(float);
898 -
899 - dim3 block(NUM_THREADS);
900 - const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
901 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
902 - switch (head_size) {
903 - // NOTE(woosuk): To reduce the compilation time, we only compile for the
904 - // head sizes that we use in the model. However, we can easily extend this
905 - // to support any head size which is a multiple of 16.
906 - case 64:
907 - LAUNCH_PAGED_ATTENTION_V2(64);
908 - break;
909 - case 80:
910 - LAUNCH_PAGED_ATTENTION_V2(80);
911 - break;
912 - case 96:
913 - LAUNCH_PAGED_ATTENTION_V2(96);
914 - break;
915 - case 112:
916 - LAUNCH_PAGED_ATTENTION_V2(112);
917 - break;
918 - case 120:
919 - LAUNCH_PAGED_ATTENTION_V2(120);
920 - break;
921 - case 128:
922 - LAUNCH_PAGED_ATTENTION_V2(128);
923 - break;
924 - case 192:
925 - LAUNCH_PAGED_ATTENTION_V2(192);
926 - break;
927 - case 256:
928 - LAUNCH_PAGED_ATTENTION_V2(256);
929 - break;
930 - default:
931 - TORCH_CHECK(false, "Unsupported head size: ", head_size);
932 - break;
933 - }
934 -}
935 -
936 -#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE, IS_BLOCK_SPARSE) \
937 - paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
938 - IS_BLOCK_SPARSE>( \
939 - out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
940 - num_kv_heads, scale, block_tables, seq_lens, max_seq_len, alibi_slopes, \
941 - k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
942 - blocksparse_vert_stride, blocksparse_block_size, \
943 - blocksparse_head_sliding_step);
944 -
945 -#define CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
946 - switch (is_block_sparse) { \
947 - case true: \
948 - CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, true); \
949 - break; \
950 - case false: \
951 - CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, false); \
952 - break; \
953 - }
954 -
955 -// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
956 -// 1, 2, 4, 64, 128, 256.
957 -#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
958 - switch (block_size) { \
959 - case 8: \
960 - CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 8, KV_DTYPE); \
961 - break; \
962 - case 16: \
963 - CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 16, KV_DTYPE); \
964 - break; \
965 - case 32: \
966 - CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 32, KV_DTYPE); \
967 - break; \
968 - default: \
969 - TORCH_CHECK(false, "Unsupported block size: ", block_size); \
970 - break; \
971 - }
972 -
973 -void paged_attention_v2(
974 - torch::Tensor& out, // [num_seqs, num_heads, head_size]
975 - torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions]
976 - torch::Tensor& max_logits, // [num_seqs, num_heads, max_num_partitions]
977 - torch::Tensor&
978 - tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
979 - torch::Tensor& query, // [num_seqs, num_heads, head_size]
980 - torch::Tensor&
981 - key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
982 - torch::Tensor&
983 - value_cache, // [num_blocks, num_heads, head_size, block_size]
984 - int64_t num_kv_heads, // [num_heads]
985 - double scale,
986 - torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
987 - torch::Tensor& seq_lens, // [num_seqs]
988 - int64_t block_size, int64_t max_seq_len,
989 - const c10::optional<torch::Tensor>& alibi_slopes,
990 - const std::string& kv_cache_dtype, double k_scale, double v_scale,
991 - const int64_t tp_rank, const int64_t blocksparse_local_blocks,
992 - const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
993 - const int64_t blocksparse_head_sliding_step) {
994 - const bool is_block_sparse = (blocksparse_vert_stride > 1);
995 - DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
996 - CALL_V2_LAUNCHER_BLOCK_SIZE)
997 -}
998 -
673 #undef WARP_SIZE
674 #undef MAX
675 #undef MIN
csrc/attention/paged_attention_v1.cu new
+193
@@ -0,0 +1,193 @@
1 +/*
2 + * Adapted from
3 + * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4 + * Copyright (c) 2023, The vLLM team.
5 + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6 + *
7 + * Licensed under the Apache License, Version 2.0 (the "License");
8 + * you may not use this file except in compliance with the License.
9 + * You may obtain a copy of the License at
10 + *
11 + * http://www.apache.org/licenses/LICENSE-2.0
12 + *
13 + * Unless required by applicable law or agreed to in writing, software
14 + * distributed under the License is distributed on an "AS IS" BASIS,
15 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 + * See the License for the specific language governing permissions and
17 + * limitations under the License.
18 + */
19 +
20 +#include "attention_kernels.cuh"
21 +
22 +#ifndef USE_ROCM
23 + #define WARP_SIZE 32
24 +#else
25 + #define WARP_SIZE warpSize
26 +#endif
27 +
28 +#define MAX(a, b) ((a) > (b) ? (a) : (b))
29 +#define MIN(a, b) ((a) < (b) ? (a) : (b))
30 +#define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
31 +
32 +#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
33 + VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
34 + ((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, \
35 + BLOCK_SIZE, NUM_THREADS, \
36 + KV_DTYPE, IS_BLOCK_SPARSE>), \
37 + shared_mem_size); \
38 + vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
39 + NUM_THREADS, KV_DTYPE, IS_BLOCK_SPARSE> \
40 + <<<grid, block, shared_mem_size, stream>>>( \
41 + out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, \
42 + scale, block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
43 + alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
44 + k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
45 + blocksparse_vert_stride, blocksparse_block_size, \
46 + blocksparse_head_sliding_step);
47 +
48 +// TODO(woosuk): Tune NUM_THREADS.
49 +template <typename T, typename CACHE_T, int BLOCK_SIZE,
50 + vllm::Fp8KVCacheDataType KV_DTYPE, bool IS_BLOCK_SPARSE,
51 + int NUM_THREADS = 128>
52 +void paged_attention_v1_launcher(
53 + torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
54 + torch::Tensor& value_cache, int num_kv_heads, float scale,
55 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
56 + const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
57 + float v_scale, const int tp_rank, const int blocksparse_local_blocks,
58 + const int blocksparse_vert_stride, const int blocksparse_block_size,
59 + const int blocksparse_head_sliding_step) {
60 + int num_seqs = query.size(0);
61 + int num_heads = query.size(1);
62 + int head_size = query.size(2);
63 + int max_num_blocks_per_seq = block_tables.size(1);
64 + int q_stride = query.stride(0);
65 + int kv_block_stride = key_cache.stride(0);
66 + int kv_head_stride = key_cache.stride(1);
67 +
68 + [[maybe_unused]] int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
69 + assert(head_size % thread_group_size == 0);
70 +
71 + // NOTE: alibi_slopes is optional.
72 + const float* alibi_slopes_ptr =
73 + alibi_slopes
74 + ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
75 + : nullptr;
76 +
77 + T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
78 + T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
79 + CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
80 + CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
81 + int* block_tables_ptr = block_tables.data_ptr<int>();
82 + int* seq_lens_ptr = seq_lens.data_ptr<int>();
83 +
84 + constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
85 + int padded_max_seq_len =
86 + DIVIDE_ROUND_UP(max_seq_len, BLOCK_SIZE) * BLOCK_SIZE;
87 + int logits_size = padded_max_seq_len * sizeof(float);
88 + int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
89 + // Python-side check in vllm.worker.worker._check_if_can_support_max_seq_len
90 + // Keep that in sync with the logic here!
91 + int shared_mem_size = std::max(logits_size, outputs_size);
92 +
93 + dim3 grid(num_heads, num_seqs, 1);
94 + dim3 block(NUM_THREADS);
95 + const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
96 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
97 + switch (head_size) {
98 + // NOTE(woosuk): To reduce the compilation time, we only compile for the
99 + // head sizes that we use in the model. However, we can easily extend this
100 + // to support any head size which is a multiple of 16.
101 + case 64:
102 + LAUNCH_PAGED_ATTENTION_V1(64);
103 + break;
104 + case 80:
105 + LAUNCH_PAGED_ATTENTION_V1(80);
106 + break;
107 + case 96:
108 + LAUNCH_PAGED_ATTENTION_V1(96);
109 + break;
110 + case 112:
111 + LAUNCH_PAGED_ATTENTION_V1(112);
112 + break;
113 + case 120:
114 + LAUNCH_PAGED_ATTENTION_V1(120);
115 + break;
116 + case 128:
117 + LAUNCH_PAGED_ATTENTION_V1(128);
118 + break;
119 + case 192:
120 + LAUNCH_PAGED_ATTENTION_V1(192);
121 + break;
122 + case 256:
123 + LAUNCH_PAGED_ATTENTION_V1(256);
124 + break;
125 + default:
126 + TORCH_CHECK(false, "Unsupported head size: ", head_size);
127 + break;
128 + }
129 +}
130 +
131 +#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE, IS_BLOCK_SPARSE) \
132 + paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
133 + IS_BLOCK_SPARSE>( \
134 + out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
135 + seq_lens, max_seq_len, alibi_slopes, k_scale, v_scale, tp_rank, \
136 + blocksparse_local_blocks, blocksparse_vert_stride, \
137 + blocksparse_block_size, blocksparse_head_sliding_step);
138 +
139 +#define CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
140 + switch (is_block_sparse) { \
141 + case true: \
142 + CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, true); \
143 + break; \
144 + case false: \
145 + CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, false); \
146 + break; \
147 + }
148 +
149 +// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
150 +// 1, 2, 4, 64, 128, 256.
151 +#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
152 + switch (block_size) { \
153 + case 8: \
154 + CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 8, KV_DTYPE); \
155 + break; \
156 + case 16: \
157 + CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 16, KV_DTYPE); \
158 + break; \
159 + case 32: \
160 + CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 32, KV_DTYPE); \
161 + break; \
162 + default: \
163 + TORCH_CHECK(false, "Unsupported block size: ", block_size); \
164 + break; \
165 + }
166 +
167 +void paged_attention_v1(
168 + torch::Tensor& out, // [num_seqs, num_heads, head_size]
169 + torch::Tensor& query, // [num_seqs, num_heads, head_size]
170 + torch::Tensor&
171 + key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
172 + torch::Tensor&
173 + value_cache, // [num_blocks, num_heads, head_size, block_size]
174 + int64_t num_kv_heads, // [num_heads]
175 + double scale,
176 + torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
177 + torch::Tensor& seq_lens, // [num_seqs]
178 + int64_t block_size, int64_t max_seq_len,
179 + const c10::optional<torch::Tensor>& alibi_slopes,
180 + const std::string& kv_cache_dtype, double k_scale, double v_scale,
181 + const int64_t tp_rank, const int64_t blocksparse_local_blocks,
182 + const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
183 + const int64_t blocksparse_head_sliding_step) {
184 + const bool is_block_sparse = (blocksparse_vert_stride > 1);
185 +
186 + DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
187 + CALL_V1_LAUNCHER_BLOCK_SIZE)
188 +}
189 +
190 +#undef WARP_SIZE
191 +#undef MAX
192 +#undef MIN
193 +#undef DIVIDE_ROUND_UP
\ No newline at end of file
csrc/attention/paged_attention_v2.cu new
+203
@@ -0,0 +1,203 @@
1 +/*
2 + * Adapted from
3 + * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4 + * Copyright (c) 2023, The vLLM team.
5 + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6 + *
7 + * Licensed under the Apache License, Version 2.0 (the "License");
8 + * you may not use this file except in compliance with the License.
9 + * You may obtain a copy of the License at
10 + *
11 + * http://www.apache.org/licenses/LICENSE-2.0
12 + *
13 + * Unless required by applicable law or agreed to in writing, software
14 + * distributed under the License is distributed on an "AS IS" BASIS,
15 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 + * See the License for the specific language governing permissions and
17 + * limitations under the License.
18 + */
19 +
20 +#include "attention_kernels.cuh"
21 +
22 +#ifndef USE_ROCM
23 + #define WARP_SIZE 32
24 +#else
25 + #define WARP_SIZE warpSize
26 +#endif
27 +
28 +#define MAX(a, b) ((a) > (b) ? (a) : (b))
29 +#define MIN(a, b) ((a) < (b) ? (a) : (b))
30 +#define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
31 +
32 +#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
33 + vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
34 + NUM_THREADS, KV_DTYPE, IS_BLOCK_SPARSE, \
35 + PARTITION_SIZE> \
36 + <<<grid, block, shared_mem_size, stream>>>( \
37 + exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, key_cache_ptr, \
38 + value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
39 + seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
40 + kv_block_stride, kv_head_stride, k_scale, v_scale, tp_rank, \
41 + blocksparse_local_blocks, blocksparse_vert_stride, \
42 + blocksparse_block_size, blocksparse_head_sliding_step); \
43 + vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, \
44 + PARTITION_SIZE> \
45 + <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
46 + out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, seq_lens_ptr, \
47 + max_num_partitions);
48 +
49 +template <typename T, typename CACHE_T, int BLOCK_SIZE,
50 + vllm::Fp8KVCacheDataType KV_DTYPE, bool IS_BLOCK_SPARSE,
51 + int NUM_THREADS = 128, int PARTITION_SIZE = 512>
52 +void paged_attention_v2_launcher(
53 + torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
54 + torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
55 + torch::Tensor& value_cache, int num_kv_heads, float scale,
56 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
57 + const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
58 + float v_scale, const int tp_rank, const int blocksparse_local_blocks,
59 + const int blocksparse_vert_stride, const int blocksparse_block_size,
60 + const int blocksparse_head_sliding_step) {
61 + int num_seqs = query.size(0);
62 + int num_heads = query.size(1);
63 + int head_size = query.size(2);
64 + int max_num_blocks_per_seq = block_tables.size(1);
65 + int q_stride = query.stride(0);
66 + int kv_block_stride = key_cache.stride(0);
67 + int kv_head_stride = key_cache.stride(1);
68 +
69 + [[maybe_unused]] int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
70 + assert(head_size % thread_group_size == 0);
71 +
72 + // NOTE: alibi_slopes is optional.
73 + const float* alibi_slopes_ptr =
74 + alibi_slopes
75 + ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
76 + : nullptr;
77 +
78 + T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
79 + float* exp_sums_ptr = reinterpret_cast<float*>(exp_sums.data_ptr());
80 + float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
81 + T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
82 + T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
83 + CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
84 + CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
85 + int* block_tables_ptr = block_tables.data_ptr<int>();
86 + int* seq_lens_ptr = seq_lens.data_ptr<int>();
87 +
88 + constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
89 + int max_num_partitions = DIVIDE_ROUND_UP(max_seq_len, PARTITION_SIZE);
90 + int logits_size = PARTITION_SIZE * sizeof(float);
91 + int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
92 +
93 + // For paged attention v2 kernel.
94 + dim3 grid(num_heads, num_seqs, max_num_partitions);
95 + int shared_mem_size = std::max(logits_size, outputs_size);
96 + // For paged attention v2 reduce kernel.
97 + dim3 reduce_grid(num_heads, num_seqs);
98 + int reduce_shared_mem_size = 2 * max_num_partitions * sizeof(float);
99 +
100 + dim3 block(NUM_THREADS);
101 + const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
102 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
103 + switch (head_size) {
104 + // NOTE(woosuk): To reduce the compilation time, we only compile for the
105 + // head sizes that we use in the model. However, we can easily extend this
106 + // to support any head size which is a multiple of 16.
107 + case 64:
108 + LAUNCH_PAGED_ATTENTION_V2(64);
109 + break;
110 + case 80:
111 + LAUNCH_PAGED_ATTENTION_V2(80);
112 + break;
113 + case 96:
114 + LAUNCH_PAGED_ATTENTION_V2(96);
115 + break;
116 + case 112:
117 + LAUNCH_PAGED_ATTENTION_V2(112);
118 + break;
119 + case 120:
120 + LAUNCH_PAGED_ATTENTION_V2(120);
121 + break;
122 + case 128:
123 + LAUNCH_PAGED_ATTENTION_V2(128);
124 + break;
125 + case 192:
126 + LAUNCH_PAGED_ATTENTION_V2(192);
127 + break;
128 + case 256:
129 + LAUNCH_PAGED_ATTENTION_V2(256);
130 + break;
131 + default:
132 + TORCH_CHECK(false, "Unsupported head size: ", head_size);
133 + break;
134 + }
135 +}
136 +
137 +#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE, IS_BLOCK_SPARSE) \
138 + paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
139 + IS_BLOCK_SPARSE>( \
140 + out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
141 + num_kv_heads, scale, block_tables, seq_lens, max_seq_len, alibi_slopes, \
142 + k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
143 + blocksparse_vert_stride, blocksparse_block_size, \
144 + blocksparse_head_sliding_step);
145 +
146 +#define CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
147 + switch (is_block_sparse) { \
148 + case true: \
149 + CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, true); \
150 + break; \
151 + case false: \
152 + CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, false); \
153 + break; \
154 + }
155 +
156 +// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
157 +// 1, 2, 4, 64, 128, 256.
158 +#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
159 + switch (block_size) { \
160 + case 8: \
161 + CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 8, KV_DTYPE); \
162 + break; \
163 + case 16: \
164 + CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 16, KV_DTYPE); \
165 + break; \
166 + case 32: \
167 + CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 32, KV_DTYPE); \
168 + break; \
169 + default: \
170 + TORCH_CHECK(false, "Unsupported block size: ", block_size); \
171 + break; \
172 + }
173 +
174 +void paged_attention_v2(
175 + torch::Tensor& out, // [num_seqs, num_heads, head_size]
176 + torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions]
177 + torch::Tensor& max_logits, // [num_seqs, num_heads, max_num_partitions]
178 + torch::Tensor&
179 + tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
180 + torch::Tensor& query, // [num_seqs, num_heads, head_size]
181 + torch::Tensor&
182 + key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
183 + torch::Tensor&
184 + value_cache, // [num_blocks, num_heads, head_size, block_size]
185 + int64_t num_kv_heads, // [num_heads]
186 + double scale,
187 + torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
188 + torch::Tensor& seq_lens, // [num_seqs]
189 + int64_t block_size, int64_t max_seq_len,
190 + const c10::optional<torch::Tensor>& alibi_slopes,
191 + const std::string& kv_cache_dtype, double k_scale, double v_scale,
192 + const int64_t tp_rank, const int64_t blocksparse_local_blocks,
193 + const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
194 + const int64_t blocksparse_head_sliding_step) {
195 + const bool is_block_sparse = (blocksparse_vert_stride > 1);
196 + DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
197 + CALL_V2_LAUNCHER_BLOCK_SIZE)
198 +}
199 +
200 +#undef WARP_SIZE
201 +#undef MAX
202 +#undef MIN
203 +#undef DIVIDE_ROUND_UP
\ No newline at end of file