A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
Splitting attention kernel file (#10091)
Signed-off-by: maleksan85 <maleksan@amd.com> Co-authored-by: Aleksandr Malyshev <maleksan@amd.com>
Aleksandr Malyshev committed
Nov 11, 2024 at 22:55 UTC
812c981fa00a8b2b95865c6e76b6c3735a56d7d9
4 files changed
+399
-328
CMakeLists.txt
+3
-2
@@ -37,7 +37,7 @@ set(PYTHON_SUPPORTED_VERSIONS "3.9" "3.10" "3.11" "3.12")
37
set(CUDA_SUPPORTED_ARCHS "7.0;7.5;8.0;8.6;8.9;9.0")
38
39
# Supported AMD GPU architectures.
40
-set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100")
40
+set(HIP_SUPPORTED_ARCHS "gfx906;gfx908;gfx90a;gfx940;gfx941;gfx942;gfx1030;gfx1100;gfx1101")
41
42
#
43
# Supported/expected torch versions for CUDA/ROCm.
@@ -187,7 +187,8 @@ message(STATUS "FetchContent base directory: ${FETCHCONTENT_BASE_DIR}")
187
188
set(VLLM_EXT_SRC
189
"csrc/cache_kernels.cu"
190
- "csrc/attention/attention_kernels.cu"
190
+ "csrc/attention/paged_attention_v1.cu"
191
+ "csrc/attention/paged_attention_v2.cu"
192
"csrc/pos_encoding_kernels.cu"
193
"csrc/activation_kernels.cu"
194
"csrc/layernorm_kernels.cu"
csrc/attention/attention_kernels.cuh
renamed
-326
@@ -670,332 +670,6 @@ __global__ void paged_attention_v2_reduce_kernel(
670
671
} // namespace vllm
672
673
-#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
674
- VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
675
- ((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, \
676
- BLOCK_SIZE, NUM_THREADS, \
677
- KV_DTYPE, IS_BLOCK_SPARSE>), \
678
- shared_mem_size); \
679
- vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
680
- NUM_THREADS, KV_DTYPE, IS_BLOCK_SPARSE> \
681
- <<<grid, block, shared_mem_size, stream>>>( \
682
- out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, \
683
- scale, block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
684
- alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
685
- k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
686
- blocksparse_vert_stride, blocksparse_block_size, \
687
- blocksparse_head_sliding_step);
688
-
689
-// TODO(woosuk): Tune NUM_THREADS.
690
-template <typename T, typename CACHE_T, int BLOCK_SIZE,
691
- vllm::Fp8KVCacheDataType KV_DTYPE, bool IS_BLOCK_SPARSE,
692
- int NUM_THREADS = 128>
693
-void paged_attention_v1_launcher(
694
- torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
695
- torch::Tensor& value_cache, int num_kv_heads, float scale,
696
- torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
697
- const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
698
- float v_scale, const int tp_rank, const int blocksparse_local_blocks,
699
- const int blocksparse_vert_stride, const int blocksparse_block_size,
700
- const int blocksparse_head_sliding_step) {
701
- int num_seqs = query.size(0);
702
- int num_heads = query.size(1);
703
- int head_size = query.size(2);
704
- int max_num_blocks_per_seq = block_tables.size(1);
705
- int q_stride = query.stride(0);
706
- int kv_block_stride = key_cache.stride(0);
707
- int kv_head_stride = key_cache.stride(1);
708
-
709
- [[maybe_unused]] int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
710
- assert(head_size % thread_group_size == 0);
711
-
712
- // NOTE: alibi_slopes is optional.
713
- const float* alibi_slopes_ptr =
714
- alibi_slopes
715
- ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
716
- : nullptr;
717
-
718
- T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
719
- T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
720
- CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
721
- CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
722
- int* block_tables_ptr = block_tables.data_ptr<int>();
723
- int* seq_lens_ptr = seq_lens.data_ptr<int>();
724
-
725
- constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
726
- int padded_max_seq_len =
727
- DIVIDE_ROUND_UP(max_seq_len, BLOCK_SIZE) * BLOCK_SIZE;
728
- int logits_size = padded_max_seq_len * sizeof(float);
729
- int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
730
- // Python-side check in vllm.worker.worker._check_if_can_support_max_seq_len
731
- // Keep that in sync with the logic here!
732
- int shared_mem_size = std::max(logits_size, outputs_size);
733
-
734
- dim3 grid(num_heads, num_seqs, 1);
735
- dim3 block(NUM_THREADS);
736
- const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
737
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
738
- switch (head_size) {
739
- // NOTE(woosuk): To reduce the compilation time, we only compile for the
740
- // head sizes that we use in the model. However, we can easily extend this
741
- // to support any head size which is a multiple of 16.
742
- case 64:
743
- LAUNCH_PAGED_ATTENTION_V1(64);
744
- break;
745
- case 80:
746
- LAUNCH_PAGED_ATTENTION_V1(80);
747
- break;
748
- case 96:
749
- LAUNCH_PAGED_ATTENTION_V1(96);
750
- break;
751
- case 112:
752
- LAUNCH_PAGED_ATTENTION_V1(112);
753
- break;
754
- case 120:
755
- LAUNCH_PAGED_ATTENTION_V1(120);
756
- break;
757
- case 128:
758
- LAUNCH_PAGED_ATTENTION_V1(128);
759
- break;
760
- case 192:
761
- LAUNCH_PAGED_ATTENTION_V1(192);
762
- break;
763
- case 256:
764
- LAUNCH_PAGED_ATTENTION_V1(256);
765
- break;
766
- default:
767
- TORCH_CHECK(false, "Unsupported head size: ", head_size);
768
- break;
769
- }
770
-}
771
-
772
-#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE, IS_BLOCK_SPARSE) \
773
- paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
774
- IS_BLOCK_SPARSE>( \
775
- out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
776
- seq_lens, max_seq_len, alibi_slopes, k_scale, v_scale, tp_rank, \
777
- blocksparse_local_blocks, blocksparse_vert_stride, \
778
- blocksparse_block_size, blocksparse_head_sliding_step);
779
-
780
-#define CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
781
- switch (is_block_sparse) { \
782
- case true: \
783
- CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, true); \
784
- break; \
785
- case false: \
786
- CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, false); \
787
- break; \
788
- }
789
-
790
-// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
791
-// 1, 2, 4, 64, 128, 256.
792
-#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
793
- switch (block_size) { \
794
- case 8: \
795
- CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 8, KV_DTYPE); \
796
- break; \
797
- case 16: \
798
- CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 16, KV_DTYPE); \
799
- break; \
800
- case 32: \
801
- CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 32, KV_DTYPE); \
802
- break; \
803
- default: \
804
- TORCH_CHECK(false, "Unsupported block size: ", block_size); \
805
- break; \
806
- }
807
-
808
-void paged_attention_v1(
809
- torch::Tensor& out, // [num_seqs, num_heads, head_size]
810
- torch::Tensor& query, // [num_seqs, num_heads, head_size]
811
- torch::Tensor&
812
- key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
813
- torch::Tensor&
814
- value_cache, // [num_blocks, num_heads, head_size, block_size]
815
- int64_t num_kv_heads, // [num_heads]
816
- double scale,
817
- torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
818
- torch::Tensor& seq_lens, // [num_seqs]
819
- int64_t block_size, int64_t max_seq_len,
820
- const c10::optional<torch::Tensor>& alibi_slopes,
821
- const std::string& kv_cache_dtype, double k_scale, double v_scale,
822
- const int64_t tp_rank, const int64_t blocksparse_local_blocks,
823
- const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
824
- const int64_t blocksparse_head_sliding_step) {
825
- const bool is_block_sparse = (blocksparse_vert_stride > 1);
826
-
827
- DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
828
- CALL_V1_LAUNCHER_BLOCK_SIZE)
829
-}
830
-
831
-#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
832
- vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
833
- NUM_THREADS, KV_DTYPE, IS_BLOCK_SPARSE, \
834
- PARTITION_SIZE> \
835
- <<<grid, block, shared_mem_size, stream>>>( \
836
- exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, key_cache_ptr, \
837
- value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
838
- seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
839
- kv_block_stride, kv_head_stride, k_scale, v_scale, tp_rank, \
840
- blocksparse_local_blocks, blocksparse_vert_stride, \
841
- blocksparse_block_size, blocksparse_head_sliding_step); \
842
- vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, \
843
- PARTITION_SIZE> \
844
- <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
845
- out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, seq_lens_ptr, \
846
- max_num_partitions);
847
-
848
-template <typename T, typename CACHE_T, int BLOCK_SIZE,
849
- vllm::Fp8KVCacheDataType KV_DTYPE, bool IS_BLOCK_SPARSE,
850
- int NUM_THREADS = 128, int PARTITION_SIZE = 512>
851
-void paged_attention_v2_launcher(
852
- torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
853
- torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
854
- torch::Tensor& value_cache, int num_kv_heads, float scale,
855
- torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
856
- const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
857
- float v_scale, const int tp_rank, const int blocksparse_local_blocks,
858
- const int blocksparse_vert_stride, const int blocksparse_block_size,
859
- const int blocksparse_head_sliding_step) {
860
- int num_seqs = query.size(0);
861
- int num_heads = query.size(1);
862
- int head_size = query.size(2);
863
- int max_num_blocks_per_seq = block_tables.size(1);
864
- int q_stride = query.stride(0);
865
- int kv_block_stride = key_cache.stride(0);
866
- int kv_head_stride = key_cache.stride(1);
867
-
868
- [[maybe_unused]] int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
869
- assert(head_size % thread_group_size == 0);
870
-
871
- // NOTE: alibi_slopes is optional.
872
- const float* alibi_slopes_ptr =
873
- alibi_slopes
874
- ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
875
- : nullptr;
876
-
877
- T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
878
- float* exp_sums_ptr = reinterpret_cast<float*>(exp_sums.data_ptr());
879
- float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
880
- T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
881
- T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
882
- CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
883
- CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
884
- int* block_tables_ptr = block_tables.data_ptr<int>();
885
- int* seq_lens_ptr = seq_lens.data_ptr<int>();
886
-
887
- constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
888
- int max_num_partitions = DIVIDE_ROUND_UP(max_seq_len, PARTITION_SIZE);
889
- int logits_size = PARTITION_SIZE * sizeof(float);
890
- int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
891
-
892
- // For paged attention v2 kernel.
893
- dim3 grid(num_heads, num_seqs, max_num_partitions);
894
- int shared_mem_size = std::max(logits_size, outputs_size);
895
- // For paged attention v2 reduce kernel.
896
- dim3 reduce_grid(num_heads, num_seqs);
897
- int reduce_shared_mem_size = 2 * max_num_partitions * sizeof(float);
898
-
899
- dim3 block(NUM_THREADS);
900
- const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
901
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
902
- switch (head_size) {
903
- // NOTE(woosuk): To reduce the compilation time, we only compile for the
904
- // head sizes that we use in the model. However, we can easily extend this
905
- // to support any head size which is a multiple of 16.
906
- case 64:
907
- LAUNCH_PAGED_ATTENTION_V2(64);
908
- break;
909
- case 80:
910
- LAUNCH_PAGED_ATTENTION_V2(80);
911
- break;
912
- case 96:
913
- LAUNCH_PAGED_ATTENTION_V2(96);
914
- break;
915
- case 112:
916
- LAUNCH_PAGED_ATTENTION_V2(112);
917
- break;
918
- case 120:
919
- LAUNCH_PAGED_ATTENTION_V2(120);
920
- break;
921
- case 128:
922
- LAUNCH_PAGED_ATTENTION_V2(128);
923
- break;
924
- case 192:
925
- LAUNCH_PAGED_ATTENTION_V2(192);
926
- break;
927
- case 256:
928
- LAUNCH_PAGED_ATTENTION_V2(256);
929
- break;
930
- default:
931
- TORCH_CHECK(false, "Unsupported head size: ", head_size);
932
- break;
933
- }
934
-}
935
-
936
-#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE, IS_BLOCK_SPARSE) \
937
- paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
938
- IS_BLOCK_SPARSE>( \
939
- out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
940
- num_kv_heads, scale, block_tables, seq_lens, max_seq_len, alibi_slopes, \
941
- k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
942
- blocksparse_vert_stride, blocksparse_block_size, \
943
- blocksparse_head_sliding_step);
944
-
945
-#define CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
946
- switch (is_block_sparse) { \
947
- case true: \
948
- CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, true); \
949
- break; \
950
- case false: \
951
- CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, false); \
952
- break; \
953
- }
954
-
955
-// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
956
-// 1, 2, 4, 64, 128, 256.
957
-#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
958
- switch (block_size) { \
959
- case 8: \
960
- CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 8, KV_DTYPE); \
961
- break; \
962
- case 16: \
963
- CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 16, KV_DTYPE); \
964
- break; \
965
- case 32: \
966
- CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 32, KV_DTYPE); \
967
- break; \
968
- default: \
969
- TORCH_CHECK(false, "Unsupported block size: ", block_size); \
970
- break; \
971
- }
972
-
973
-void paged_attention_v2(
974
- torch::Tensor& out, // [num_seqs, num_heads, head_size]
975
- torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions]
976
- torch::Tensor& max_logits, // [num_seqs, num_heads, max_num_partitions]
977
- torch::Tensor&
978
- tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
979
- torch::Tensor& query, // [num_seqs, num_heads, head_size]
980
- torch::Tensor&
981
- key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
982
- torch::Tensor&
983
- value_cache, // [num_blocks, num_heads, head_size, block_size]
984
- int64_t num_kv_heads, // [num_heads]
985
- double scale,
986
- torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
987
- torch::Tensor& seq_lens, // [num_seqs]
988
- int64_t block_size, int64_t max_seq_len,
989
- const c10::optional<torch::Tensor>& alibi_slopes,
990
- const std::string& kv_cache_dtype, double k_scale, double v_scale,
991
- const int64_t tp_rank, const int64_t blocksparse_local_blocks,
992
- const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
993
- const int64_t blocksparse_head_sliding_step) {
994
- const bool is_block_sparse = (blocksparse_vert_stride > 1);
995
- DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
996
- CALL_V2_LAUNCHER_BLOCK_SIZE)
997
-}
998
-
673
#undef WARP_SIZE
674
#undef MAX
675
#undef MIN
csrc/attention/paged_attention_v1.cu
new
+193
@@ -0,0 +1,193 @@
1
+/*
2
+ * Adapted from
3
+ * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4
+ * Copyright (c) 2023, The vLLM team.
5
+ * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6
+ *
7
+ * Licensed under the Apache License, Version 2.0 (the "License");
8
+ * you may not use this file except in compliance with the License.
9
+ * You may obtain a copy of the License at
10
+ *
11
+ * http://www.apache.org/licenses/LICENSE-2.0
12
+ *
13
+ * Unless required by applicable law or agreed to in writing, software
14
+ * distributed under the License is distributed on an "AS IS" BASIS,
15
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ * See the License for the specific language governing permissions and
17
+ * limitations under the License.
18
+ */
19
+
20
+#include "attention_kernels.cuh"
21
+
22
+#ifndef USE_ROCM
23
+ #define WARP_SIZE 32
24
+#else
25
+ #define WARP_SIZE warpSize
26
+#endif
27
+
28
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
29
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
30
+#define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
31
+
32
+#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
33
+ VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
34
+ ((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, \
35
+ BLOCK_SIZE, NUM_THREADS, \
36
+ KV_DTYPE, IS_BLOCK_SPARSE>), \
37
+ shared_mem_size); \
38
+ vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
39
+ NUM_THREADS, KV_DTYPE, IS_BLOCK_SPARSE> \
40
+ <<<grid, block, shared_mem_size, stream>>>( \
41
+ out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, \
42
+ scale, block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
43
+ alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
44
+ k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
45
+ blocksparse_vert_stride, blocksparse_block_size, \
46
+ blocksparse_head_sliding_step);
47
+
48
+// TODO(woosuk): Tune NUM_THREADS.
49
+template <typename T, typename CACHE_T, int BLOCK_SIZE,
50
+ vllm::Fp8KVCacheDataType KV_DTYPE, bool IS_BLOCK_SPARSE,
51
+ int NUM_THREADS = 128>
52
+void paged_attention_v1_launcher(
53
+ torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
54
+ torch::Tensor& value_cache, int num_kv_heads, float scale,
55
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
56
+ const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
57
+ float v_scale, const int tp_rank, const int blocksparse_local_blocks,
58
+ const int blocksparse_vert_stride, const int blocksparse_block_size,
59
+ const int blocksparse_head_sliding_step) {
60
+ int num_seqs = query.size(0);
61
+ int num_heads = query.size(1);
62
+ int head_size = query.size(2);
63
+ int max_num_blocks_per_seq = block_tables.size(1);
64
+ int q_stride = query.stride(0);
65
+ int kv_block_stride = key_cache.stride(0);
66
+ int kv_head_stride = key_cache.stride(1);
67
+
68
+ [[maybe_unused]] int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
69
+ assert(head_size % thread_group_size == 0);
70
+
71
+ // NOTE: alibi_slopes is optional.
72
+ const float* alibi_slopes_ptr =
73
+ alibi_slopes
74
+ ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
75
+ : nullptr;
76
+
77
+ T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
78
+ T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
79
+ CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
80
+ CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
81
+ int* block_tables_ptr = block_tables.data_ptr<int>();
82
+ int* seq_lens_ptr = seq_lens.data_ptr<int>();
83
+
84
+ constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
85
+ int padded_max_seq_len =
86
+ DIVIDE_ROUND_UP(max_seq_len, BLOCK_SIZE) * BLOCK_SIZE;
87
+ int logits_size = padded_max_seq_len * sizeof(float);
88
+ int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
89
+ // Python-side check in vllm.worker.worker._check_if_can_support_max_seq_len
90
+ // Keep that in sync with the logic here!
91
+ int shared_mem_size = std::max(logits_size, outputs_size);
92
+
93
+ dim3 grid(num_heads, num_seqs, 1);
94
+ dim3 block(NUM_THREADS);
95
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
96
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
97
+ switch (head_size) {
98
+ // NOTE(woosuk): To reduce the compilation time, we only compile for the
99
+ // head sizes that we use in the model. However, we can easily extend this
100
+ // to support any head size which is a multiple of 16.
101
+ case 64:
102
+ LAUNCH_PAGED_ATTENTION_V1(64);
103
+ break;
104
+ case 80:
105
+ LAUNCH_PAGED_ATTENTION_V1(80);
106
+ break;
107
+ case 96:
108
+ LAUNCH_PAGED_ATTENTION_V1(96);
109
+ break;
110
+ case 112:
111
+ LAUNCH_PAGED_ATTENTION_V1(112);
112
+ break;
113
+ case 120:
114
+ LAUNCH_PAGED_ATTENTION_V1(120);
115
+ break;
116
+ case 128:
117
+ LAUNCH_PAGED_ATTENTION_V1(128);
118
+ break;
119
+ case 192:
120
+ LAUNCH_PAGED_ATTENTION_V1(192);
121
+ break;
122
+ case 256:
123
+ LAUNCH_PAGED_ATTENTION_V1(256);
124
+ break;
125
+ default:
126
+ TORCH_CHECK(false, "Unsupported head size: ", head_size);
127
+ break;
128
+ }
129
+}
130
+
131
+#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE, IS_BLOCK_SPARSE) \
132
+ paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
133
+ IS_BLOCK_SPARSE>( \
134
+ out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
135
+ seq_lens, max_seq_len, alibi_slopes, k_scale, v_scale, tp_rank, \
136
+ blocksparse_local_blocks, blocksparse_vert_stride, \
137
+ blocksparse_block_size, blocksparse_head_sliding_step);
138
+
139
+#define CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
140
+ switch (is_block_sparse) { \
141
+ case true: \
142
+ CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, true); \
143
+ break; \
144
+ case false: \
145
+ CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, false); \
146
+ break; \
147
+ }
148
+
149
+// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
150
+// 1, 2, 4, 64, 128, 256.
151
+#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
152
+ switch (block_size) { \
153
+ case 8: \
154
+ CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 8, KV_DTYPE); \
155
+ break; \
156
+ case 16: \
157
+ CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 16, KV_DTYPE); \
158
+ break; \
159
+ case 32: \
160
+ CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 32, KV_DTYPE); \
161
+ break; \
162
+ default: \
163
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
164
+ break; \
165
+ }
166
+
167
+void paged_attention_v1(
168
+ torch::Tensor& out, // [num_seqs, num_heads, head_size]
169
+ torch::Tensor& query, // [num_seqs, num_heads, head_size]
170
+ torch::Tensor&
171
+ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
172
+ torch::Tensor&
173
+ value_cache, // [num_blocks, num_heads, head_size, block_size]
174
+ int64_t num_kv_heads, // [num_heads]
175
+ double scale,
176
+ torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
177
+ torch::Tensor& seq_lens, // [num_seqs]
178
+ int64_t block_size, int64_t max_seq_len,
179
+ const c10::optional<torch::Tensor>& alibi_slopes,
180
+ const std::string& kv_cache_dtype, double k_scale, double v_scale,
181
+ const int64_t tp_rank, const int64_t blocksparse_local_blocks,
182
+ const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
183
+ const int64_t blocksparse_head_sliding_step) {
184
+ const bool is_block_sparse = (blocksparse_vert_stride > 1);
185
+
186
+ DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
187
+ CALL_V1_LAUNCHER_BLOCK_SIZE)
188
+}
189
+
190
+#undef WARP_SIZE
191
+#undef MAX
192
+#undef MIN
193
+#undef DIVIDE_ROUND_UP
\ No newline at end of file
csrc/attention/paged_attention_v2.cu
new
+203
@@ -0,0 +1,203 @@
1
+/*
2
+ * Adapted from
3
+ * https://github.com/NVIDIA/FasterTransformer/blob/release/v5.3_tag/src/fastertransformer/kernels/decoder_masked_multihead_attention/decoder_masked_multihead_attention_template.hpp
4
+ * Copyright (c) 2023, The vLLM team.
5
+ * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved.
6
+ *
7
+ * Licensed under the Apache License, Version 2.0 (the "License");
8
+ * you may not use this file except in compliance with the License.
9
+ * You may obtain a copy of the License at
10
+ *
11
+ * http://www.apache.org/licenses/LICENSE-2.0
12
+ *
13
+ * Unless required by applicable law or agreed to in writing, software
14
+ * distributed under the License is distributed on an "AS IS" BASIS,
15
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ * See the License for the specific language governing permissions and
17
+ * limitations under the License.
18
+ */
19
+
20
+#include "attention_kernels.cuh"
21
+
22
+#ifndef USE_ROCM
23
+ #define WARP_SIZE 32
24
+#else
25
+ #define WARP_SIZE warpSize
26
+#endif
27
+
28
+#define MAX(a, b) ((a) > (b) ? (a) : (b))
29
+#define MIN(a, b) ((a) < (b) ? (a) : (b))
30
+#define DIVIDE_ROUND_UP(a, b) (((a) + (b) - 1) / (b))
31
+
32
+#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
33
+ vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
34
+ NUM_THREADS, KV_DTYPE, IS_BLOCK_SPARSE, \
35
+ PARTITION_SIZE> \
36
+ <<<grid, block, shared_mem_size, stream>>>( \
37
+ exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, key_cache_ptr, \
38
+ value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
39
+ seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
40
+ kv_block_stride, kv_head_stride, k_scale, v_scale, tp_rank, \
41
+ blocksparse_local_blocks, blocksparse_vert_stride, \
42
+ blocksparse_block_size, blocksparse_head_sliding_step); \
43
+ vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, \
44
+ PARTITION_SIZE> \
45
+ <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
46
+ out_ptr, exp_sums_ptr, max_logits_ptr, tmp_out_ptr, seq_lens_ptr, \
47
+ max_num_partitions);
48
+
49
+template <typename T, typename CACHE_T, int BLOCK_SIZE,
50
+ vllm::Fp8KVCacheDataType KV_DTYPE, bool IS_BLOCK_SPARSE,
51
+ int NUM_THREADS = 128, int PARTITION_SIZE = 512>
52
+void paged_attention_v2_launcher(
53
+ torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
54
+ torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
55
+ torch::Tensor& value_cache, int num_kv_heads, float scale,
56
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
57
+ const c10::optional<torch::Tensor>& alibi_slopes, float k_scale,
58
+ float v_scale, const int tp_rank, const int blocksparse_local_blocks,
59
+ const int blocksparse_vert_stride, const int blocksparse_block_size,
60
+ const int blocksparse_head_sliding_step) {
61
+ int num_seqs = query.size(0);
62
+ int num_heads = query.size(1);
63
+ int head_size = query.size(2);
64
+ int max_num_blocks_per_seq = block_tables.size(1);
65
+ int q_stride = query.stride(0);
66
+ int kv_block_stride = key_cache.stride(0);
67
+ int kv_head_stride = key_cache.stride(1);
68
+
69
+ [[maybe_unused]] int thread_group_size = MAX(WARP_SIZE / BLOCK_SIZE, 1);
70
+ assert(head_size % thread_group_size == 0);
71
+
72
+ // NOTE: alibi_slopes is optional.
73
+ const float* alibi_slopes_ptr =
74
+ alibi_slopes
75
+ ? reinterpret_cast<const float*>(alibi_slopes.value().data_ptr())
76
+ : nullptr;
77
+
78
+ T* out_ptr = reinterpret_cast<T*>(out.data_ptr());
79
+ float* exp_sums_ptr = reinterpret_cast<float*>(exp_sums.data_ptr());
80
+ float* max_logits_ptr = reinterpret_cast<float*>(max_logits.data_ptr());
81
+ T* tmp_out_ptr = reinterpret_cast<T*>(tmp_out.data_ptr());
82
+ T* query_ptr = reinterpret_cast<T*>(query.data_ptr());
83
+ CACHE_T* key_cache_ptr = reinterpret_cast<CACHE_T*>(key_cache.data_ptr());
84
+ CACHE_T* value_cache_ptr = reinterpret_cast<CACHE_T*>(value_cache.data_ptr());
85
+ int* block_tables_ptr = block_tables.data_ptr<int>();
86
+ int* seq_lens_ptr = seq_lens.data_ptr<int>();
87
+
88
+ constexpr int NUM_WARPS = NUM_THREADS / WARP_SIZE;
89
+ int max_num_partitions = DIVIDE_ROUND_UP(max_seq_len, PARTITION_SIZE);
90
+ int logits_size = PARTITION_SIZE * sizeof(float);
91
+ int outputs_size = (NUM_WARPS / 2) * head_size * sizeof(float);
92
+
93
+ // For paged attention v2 kernel.
94
+ dim3 grid(num_heads, num_seqs, max_num_partitions);
95
+ int shared_mem_size = std::max(logits_size, outputs_size);
96
+ // For paged attention v2 reduce kernel.
97
+ dim3 reduce_grid(num_heads, num_seqs);
98
+ int reduce_shared_mem_size = 2 * max_num_partitions * sizeof(float);
99
+
100
+ dim3 block(NUM_THREADS);
101
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
102
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
103
+ switch (head_size) {
104
+ // NOTE(woosuk): To reduce the compilation time, we only compile for the
105
+ // head sizes that we use in the model. However, we can easily extend this
106
+ // to support any head size which is a multiple of 16.
107
+ case 64:
108
+ LAUNCH_PAGED_ATTENTION_V2(64);
109
+ break;
110
+ case 80:
111
+ LAUNCH_PAGED_ATTENTION_V2(80);
112
+ break;
113
+ case 96:
114
+ LAUNCH_PAGED_ATTENTION_V2(96);
115
+ break;
116
+ case 112:
117
+ LAUNCH_PAGED_ATTENTION_V2(112);
118
+ break;
119
+ case 120:
120
+ LAUNCH_PAGED_ATTENTION_V2(120);
121
+ break;
122
+ case 128:
123
+ LAUNCH_PAGED_ATTENTION_V2(128);
124
+ break;
125
+ case 192:
126
+ LAUNCH_PAGED_ATTENTION_V2(192);
127
+ break;
128
+ case 256:
129
+ LAUNCH_PAGED_ATTENTION_V2(256);
130
+ break;
131
+ default:
132
+ TORCH_CHECK(false, "Unsupported head size: ", head_size);
133
+ break;
134
+ }
135
+}
136
+
137
+#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE, IS_BLOCK_SPARSE) \
138
+ paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
139
+ IS_BLOCK_SPARSE>( \
140
+ out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
141
+ num_kv_heads, scale, block_tables, seq_lens, max_seq_len, alibi_slopes, \
142
+ k_scale, v_scale, tp_rank, blocksparse_local_blocks, \
143
+ blocksparse_vert_stride, blocksparse_block_size, \
144
+ blocksparse_head_sliding_step);
145
+
146
+#define CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
147
+ switch (is_block_sparse) { \
148
+ case true: \
149
+ CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, true); \
150
+ break; \
151
+ case false: \
152
+ CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, false); \
153
+ break; \
154
+ }
155
+
156
+// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
157
+// 1, 2, 4, 64, 128, 256.
158
+#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
159
+ switch (block_size) { \
160
+ case 8: \
161
+ CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 8, KV_DTYPE); \
162
+ break; \
163
+ case 16: \
164
+ CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 16, KV_DTYPE); \
165
+ break; \
166
+ case 32: \
167
+ CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 32, KV_DTYPE); \
168
+ break; \
169
+ default: \
170
+ TORCH_CHECK(false, "Unsupported block size: ", block_size); \
171
+ break; \
172
+ }
173
+
174
+void paged_attention_v2(
175
+ torch::Tensor& out, // [num_seqs, num_heads, head_size]
176
+ torch::Tensor& exp_sums, // [num_seqs, num_heads, max_num_partitions]
177
+ torch::Tensor& max_logits, // [num_seqs, num_heads, max_num_partitions]
178
+ torch::Tensor&
179
+ tmp_out, // [num_seqs, num_heads, max_num_partitions, head_size]
180
+ torch::Tensor& query, // [num_seqs, num_heads, head_size]
181
+ torch::Tensor&
182
+ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
183
+ torch::Tensor&
184
+ value_cache, // [num_blocks, num_heads, head_size, block_size]
185
+ int64_t num_kv_heads, // [num_heads]
186
+ double scale,
187
+ torch::Tensor& block_tables, // [num_seqs, max_num_blocks_per_seq]
188
+ torch::Tensor& seq_lens, // [num_seqs]
189
+ int64_t block_size, int64_t max_seq_len,
190
+ const c10::optional<torch::Tensor>& alibi_slopes,
191
+ const std::string& kv_cache_dtype, double k_scale, double v_scale,
192
+ const int64_t tp_rank, const int64_t blocksparse_local_blocks,
193
+ const int64_t blocksparse_vert_stride, const int64_t blocksparse_block_size,
194
+ const int64_t blocksparse_head_sliding_step) {
195
+ const bool is_block_sparse = (blocksparse_vert_stride > 1);
196
+ DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
197
+ CALL_V2_LAUNCHER_BLOCK_SIZE)
198
+}
199
+
200
+#undef WARP_SIZE
201
+#undef MAX
202
+#undef MIN
203
+#undef DIVIDE_ROUND_UP
\ No newline at end of file