@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8e192ff967
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel][Backend][Model] Blocksparse flash attention kernel and Phi-3-Small model (#4799)

Co-authored-by: beagleski <yunanzhang@microsoft.com> Co-authored-by: bapatra <bapatra@microsoft.com> Co-authored-by: Barun Patra <codedecde@users.noreply.github.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>

Eric Xihui Lin committed May 25, 2024 at 01:00 UTC 8e192ff967b44b186ea02d30e49fddf656fdfe50
23 files changed +2446 -88
csrc/attention/attention_kernels.cu
+148 -37
@@ -85,6 +85,7 @@ inline __device__ float block_sum(float* red_smem, float sum) {
85 // Grid: (num_heads, num_seqs, max_num_partitions).
86 template <typename scalar_t, typename cache_t, int HEAD_SIZE, int BLOCK_SIZE,
87 int NUM_THREADS, vllm::Fp8KVCacheDataType KV_DTYPE,
88 + bool IS_BLOCK_SPARSE,
89 int PARTITION_SIZE = 0> // Zero means no partitioning.
90 __device__ void paged_attention_kernel(
91 float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
@@ -104,7 +105,9 @@ __device__ void paged_attention_kernel(
105 const int max_num_blocks_per_seq,
106 const float* __restrict__ alibi_slopes, // [num_heads]
107 const int q_stride, const int kv_block_stride, const int kv_head_stride,
107 - const float kv_scale) {
108 + const float kv_scale, const int tp_rank, const int blocksparse_local_blocks,
109 + const int blocksparse_vert_stride, const int blocksparse_block_size,
110 + const int blocksparse_head_sliding_step) {
111 const int seq_idx = blockIdx.y;
112 const int partition_idx = blockIdx.z;
113 const int max_num_partitions = gridDim.z;
@@ -202,11 +205,55 @@ __device__ void paged_attention_kernel(
205 // Each thread group in a warp fetches a key from the block, and computes
206 // dot product with the query.
207 const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
208 +
209 + // blocksparse specific vars
210 + int bs_block_offset;
211 + int q_bs_block_id;
212 + if constexpr (IS_BLOCK_SPARSE) {
213 + // const int num_blocksparse_blocks = DIVIDE_ROUND_UP(seq_len,
214 + // blocksparse_block_size);
215 + q_bs_block_id = (seq_len - 1) / blocksparse_block_size;
216 + if (blocksparse_head_sliding_step >= 0)
217 + // sliding on q heads
218 + bs_block_offset =
219 + (tp_rank * num_heads + head_idx) * blocksparse_head_sliding_step + 1;
220 + else
221 + // sliding on kv heads
222 + bs_block_offset = (tp_rank * num_kv_heads + kv_head_idx) *
223 + (-blocksparse_head_sliding_step) +
224 + 1;
225 + }
226 +
227 for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx;
228 block_idx += NUM_WARPS) {
229 // NOTE(woosuk): The block number is stored in int32. However, we cast it to
230 // int64 because int32 can lead to overflow when this variable is multiplied
231 // by large numbers (e.g., kv_block_stride).
232 + // For blocksparse attention: skip computation on blocks that are not
233 + // attended
234 + if constexpr (IS_BLOCK_SPARSE) {
235 + const int k_bs_block_id = block_idx * BLOCK_SIZE / blocksparse_block_size;
236 + const bool is_remote =
237 + ((k_bs_block_id + bs_block_offset) % blocksparse_vert_stride == 0);
238 + const bool is_local =
239 + (k_bs_block_id > q_bs_block_id - blocksparse_local_blocks);
240 + if (!is_remote && !is_local) {
241 + for (int i = 0; i < NUM_TOKENS_PER_THREAD_GROUP; i++) {
242 + const int physical_block_offset =
243 + (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;
244 + const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
245 +
246 + if (thread_group_offset == 0) {
247 + // NOTE(linxihui): assign very large number to skipped tokens to
248 + // avoid contribution to the sumexp softmax normalizer. This will
249 + // not be used at computing sum(softmax*v) as the blocks will be
250 + // skipped.
251 + logits[token_idx - start_token_idx] = -FLT_MAX;
252 + }
253 + }
254 + continue;
255 + }
256 + }
257 const int64_t physical_block_number =
258 static_cast<int64_t>(block_table[block_idx]);
259
@@ -335,6 +382,15 @@ __device__ void paged_attention_kernel(
382 // NOTE(woosuk): The block number is stored in int32. However, we cast it to
383 // int64 because int32 can lead to overflow when this variable is multiplied
384 // by large numbers (e.g., kv_block_stride).
385 + // For blocksparse attention: skip computation on blocks that are not
386 + // attended
387 + if constexpr (IS_BLOCK_SPARSE) {
388 + int v_bs_block_id = block_idx * BLOCK_SIZE / blocksparse_block_size;
389 + if (!((v_bs_block_id + bs_block_offset) % blocksparse_vert_stride == 0) &&
390 + !((v_bs_block_id > q_bs_block_id - blocksparse_local_blocks))) {
391 + continue;
392 + }
393 + }
394 const int64_t physical_block_number =
395 static_cast<int64_t>(block_table[block_idx]);
396 const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
@@ -441,8 +497,8 @@ __device__ void paged_attention_kernel(
497
498 // Grid: (num_heads, num_seqs, 1).
499 template <typename scalar_t, typename cache_t, int HEAD_SIZE, int BLOCK_SIZE,
444 - int NUM_THREADS,
445 - vllm::Fp8KVCacheDataType KV_DTYPE>
500 + int NUM_THREADS, vllm::Fp8KVCacheDataType KV_DTYPE,
501 + bool IS_BLOCK_SPARSE>
502 __global__ void paged_attention_v1_kernel(
503 scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
504 const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
@@ -457,18 +513,23 @@ __global__ void paged_attention_v1_kernel(
513 const int max_num_blocks_per_seq,
514 const float* __restrict__ alibi_slopes, // [num_heads]
515 const int q_stride, const int kv_block_stride, const int kv_head_stride,
460 - const float kv_scale) {
516 + const float kv_scale, const int tp_rank, const int blocksparse_local_blocks,
517 + const int blocksparse_vert_stride, const int blocksparse_block_size,
518 + const int blocksparse_head_sliding_step) {
519 paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
462 - KV_DTYPE>(
520 + KV_DTYPE, IS_BLOCK_SPARSE>(
521 /* exp_sums */ nullptr, /* max_logits */ nullptr, out, q, k_cache,
522 v_cache, num_kv_heads, scale, block_tables, seq_lens,
523 max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride,
466 - kv_head_stride, kv_scale);
524 + kv_head_stride, kv_scale, tp_rank, blocksparse_local_blocks,
525 + blocksparse_vert_stride, blocksparse_block_size,
526 + blocksparse_head_sliding_step);
527 }
528
529 // Grid: (num_heads, num_seqs, max_num_partitions).
530 template <typename scalar_t, typename cache_t, int HEAD_SIZE, int BLOCK_SIZE,
531 int NUM_THREADS, vllm::Fp8KVCacheDataType KV_DTYPE,
532 + bool IS_BLOCK_SPARSE,
533 int PARTITION_SIZE>
534 __global__ void paged_attention_v2_kernel(
535 float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
@@ -488,12 +549,16 @@ __global__ void paged_attention_v2_kernel(
549 const int max_num_blocks_per_seq,
550 const float* __restrict__ alibi_slopes, // [num_heads]
551 const int q_stride, const int kv_block_stride, const int kv_head_stride,
491 - const float kv_scale) {
552 + const float kv_scale, const int tp_rank, const int blocksparse_local_blocks,
553 + const int blocksparse_vert_stride, const int blocksparse_block_size,
554 + const int blocksparse_head_sliding_step) {
555 paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
493 - KV_DTYPE, PARTITION_SIZE>(
556 + KV_DTYPE, IS_BLOCK_SPARSE, PARTITION_SIZE>(
557 exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
558 block_tables, seq_lens, max_num_blocks_per_seq, alibi_slopes, q_stride,
496 - kv_block_stride, kv_head_stride, kv_scale);
559 + kv_block_stride, kv_head_stride, kv_scale, tp_rank,
560 + blocksparse_local_blocks, blocksparse_vert_stride, blocksparse_block_size,
561 + blocksparse_head_sliding_step);
562 }
563
564 // Grid: (num_heads, num_seqs).
@@ -607,25 +672,32 @@ __global__ void paged_attention_v2_reduce_kernel(
672
673 #define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
674 VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
610 - ((void*)vllm::paged_attention_v1_kernel< \
611 - T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, KV_DTYPE>), \
675 + ((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, \
676 + BLOCK_SIZE, NUM_THREADS, \
677 + KV_DTYPE, IS_BLOCK_SPARSE>), \
678 shared_mem_size); \
679 vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
614 - NUM_THREADS, KV_DTYPE> \
680 + NUM_THREADS, KV_DTYPE, IS_BLOCK_SPARSE> \
681 <<<grid, block, shared_mem_size, stream>>>( \
682 out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, \
683 scale, block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
684 alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
619 - kv_scale);
685 + kv_scale, tp_rank, blocksparse_local_blocks, \
686 + blocksparse_vert_stride, blocksparse_block_size, \
687 + blocksparse_head_sliding_step);
688
689 // TODO(woosuk): Tune NUM_THREADS.
690 template <typename T, typename CACHE_T, int BLOCK_SIZE,
623 - vllm::Fp8KVCacheDataType KV_DTYPE, int NUM_THREADS = 128>
691 + vllm::Fp8KVCacheDataType KV_DTYPE, bool IS_BLOCK_SPARSE,
692 + int NUM_THREADS = 128>
693 void paged_attention_v1_launcher(
694 torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
695 torch::Tensor& value_cache, int num_kv_heads, float scale,
696 torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
628 - const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale) {
697 + const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale,
698 + const int tp_rank, const int blocksparse_local_blocks,
699 + const int blocksparse_vert_stride, const int blocksparse_block_size,
700 + const int blocksparse_head_sliding_step) {
701 int num_seqs = query.size(0);
702 int num_heads = query.size(1);
703 int head_size = query.size(2);
@@ -691,23 +763,36 @@ void paged_attention_v1_launcher(
763 }
764 }
765
694 -#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
695 - paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
766 +#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE, IS_BLOCK_SPARSE) \
767 + paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
768 + IS_BLOCK_SPARSE>( \
769 out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
697 - seq_lens, max_seq_len, alibi_slopes, kv_scale);
770 + seq_lens, max_seq_len, alibi_slopes, kv_scale, tp_rank, \
771 + blocksparse_local_blocks, blocksparse_vert_stride, \
772 + blocksparse_block_size, blocksparse_head_sliding_step);
773 +
774 +#define CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
775 + switch (is_block_sparse) { \
776 + case true: \
777 + CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, true); \
778 + break; \
779 + case false: \
780 + CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, false); \
781 + break; \
782 + }
783
784 // NOTE(woosuk): To reduce the compilation time, we omitted block sizes
785 // 1, 2, 4, 64, 128, 256.
786 #define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
787 switch (block_size) { \
788 case 8: \
704 - CALL_V1_LAUNCHER(T, CACHE_T, 8, KV_DTYPE); \
789 + CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 8, KV_DTYPE); \
790 break; \
791 case 16: \
707 - CALL_V1_LAUNCHER(T, CACHE_T, 16, KV_DTYPE); \
792 + CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 16, KV_DTYPE); \
793 break; \
794 case 32: \
710 - CALL_V1_LAUNCHER(T, CACHE_T, 32, KV_DTYPE); \
795 + CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 32, KV_DTYPE); \
796 break; \
797 default: \
798 TORCH_CHECK(false, "Unsupported block size: ", block_size); \
@@ -727,18 +812,26 @@ void paged_attention_v1(
812 torch::Tensor& seq_lens, // [num_seqs]
813 int block_size, int max_seq_len,
814 const c10::optional<torch::Tensor>& alibi_slopes,
730 - const std::string& kv_cache_dtype, float kv_scale){
815 + const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
816 + const int blocksparse_local_blocks, const int blocksparse_vert_stride,
817 + const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
818 + const bool is_block_sparse = (blocksparse_vert_stride > 1);
819 +
820 + DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
821 + CALL_V1_LAUNCHER_BLOCK_SIZE)
822 +}
823
732 - DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
733 - CALL_V1_LAUNCHER_BLOCK_SIZE)}
824 #define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
825 vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
736 - NUM_THREADS, KV_DTYPE, PARTITION_SIZE> \
826 + NUM_THREADS, KV_DTYPE, IS_BLOCK_SPARSE, \
827 + PARTITION_SIZE> \
828 <<<grid, block, shared_mem_size, stream>>>( \
829 exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, key_cache_ptr, \
830 value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
831 seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
741 - kv_block_stride, kv_head_stride, kv_scale); \
832 + kv_block_stride, kv_head_stride, kv_scale, tp_rank, \
833 + blocksparse_local_blocks, blocksparse_vert_stride, \
834 + blocksparse_block_size, blocksparse_head_sliding_step); \
835 vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, \
836 PARTITION_SIZE> \
837 <<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
@@ -746,14 +839,17 @@ void paged_attention_v1(
839 max_num_partitions);
840
841 template <typename T, typename CACHE_T, int BLOCK_SIZE,
749 - vllm::Fp8KVCacheDataType KV_DTYPE, int NUM_THREADS = 128,
750 - int PARTITION_SIZE = 512>
842 + vllm::Fp8KVCacheDataType KV_DTYPE, bool IS_BLOCK_SPARSE,
843 + int NUM_THREADS = 128, int PARTITION_SIZE = 512>
844 void paged_attention_v2_launcher(
845 torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
846 torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
847 torch::Tensor& value_cache, int num_kv_heads, float scale,
848 torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
756 - const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale) {
849 + const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale,
850 + const int tp_rank, const int blocksparse_local_blocks,
851 + const int blocksparse_vert_stride, const int blocksparse_block_size,
852 + const int blocksparse_head_sliding_step) {
853 int num_seqs = query.size(0);
854 int num_heads = query.size(1);
855 int head_size = query.size(2);
@@ -824,24 +920,36 @@ void paged_attention_v2_launcher(
920 }
921 }
922
827 -#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
828 - paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
923 +#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE, IS_BLOCK_SPARSE) \
924 + paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
925 + IS_BLOCK_SPARSE>( \
926 out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
927 num_kv_heads, scale, block_tables, seq_lens, max_seq_len, alibi_slopes, \
831 - kv_scale);
928 + kv_scale, tp_rank, blocksparse_local_blocks, blocksparse_vert_stride, \
929 + blocksparse_block_size, blocksparse_head_sliding_step);
930 +
931 +#define CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
932 + switch (is_block_sparse) { \
933 + case true: \
934 + CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, true); \
935 + break; \
936 + case false: \
937 + CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, false); \
938 + break; \
939 + }
940
941 // NOTE(woosuk): To reduce the compilation time, we omitted block sizes
942 // 1, 2, 4, 64, 128, 256.
943 #define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
944 switch (block_size) { \
945 case 8: \
838 - CALL_V2_LAUNCHER(T, CACHE_T, 8, KV_DTYPE); \
946 + CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 8, KV_DTYPE); \
947 break; \
948 case 16: \
841 - CALL_V2_LAUNCHER(T, CACHE_T, 16, KV_DTYPE); \
949 + CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 16, KV_DTYPE); \
950 break; \
951 case 32: \
844 - CALL_V2_LAUNCHER(T, CACHE_T, 32, KV_DTYPE); \
952 + CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 32, KV_DTYPE); \
953 break; \
954 default: \
955 TORCH_CHECK(false, "Unsupported block size: ", block_size); \
@@ -865,7 +973,10 @@ void paged_attention_v2(
973 torch::Tensor& seq_lens, // [num_seqs]
974 int block_size, int max_seq_len,
975 const c10::optional<torch::Tensor>& alibi_slopes,
868 - const std::string& kv_cache_dtype, float kv_scale) {
976 + const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
977 + const int blocksparse_local_blocks, const int blocksparse_vert_stride,
978 + const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
979 + const bool is_block_sparse = (blocksparse_vert_stride > 1);
980 DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
981 CALL_V2_LAUNCHER_BLOCK_SIZE)
982 }
@@ -873,4 +984,4 @@ void paged_attention_v2(
984 #undef WARP_SIZE
985 #undef MAX
986 #undef MIN
876 -#undef DIVIDE_ROUND_UP
987 +#undef DIVIDE_ROUND_UP
\ No newline at end of file
csrc/cpu/attention.cpp
+21 -16
@@ -415,14 +415,17 @@ void paged_attention_v1_impl_launcher(
415 }
416 } // namespace
417
418 -void paged_attention_v1(torch::Tensor& out, torch::Tensor& query,
419 - torch::Tensor& key_cache, torch::Tensor& value_cache,
420 - int num_kv_heads, float scale,
421 - torch::Tensor& block_tables, torch::Tensor& seq_lens,
422 - int block_size, int max_seq_len,
423 - const c10::optional<torch::Tensor>& alibi_slopes,
424 - const std::string& kv_cache_dtype, float kv_scale) {
418 +void paged_attention_v1(
419 + torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
420 + torch::Tensor& value_cache, int num_kv_heads, float scale,
421 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
422 + int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
423 + const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
424 + const int blocksparse_local_blocks, const int blocksparse_vert_stride,
425 + const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
426 TORCH_CHECK(kv_scale == 1.0f);
427 + TORCH_CHECK(blocksparse_vert_stride <= 1,
428 + "CPU backend does not support blocksparse attention yet.");
429 VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v1_impl",
430 [&] {
431 CPU_KERNEL_GUARD_IN(paged_attention_v1_impl)
@@ -726,16 +729,18 @@ void paged_attention_v2_impl_launcher(
729 }
730 } // namespace
731
729 -void paged_attention_v2(torch::Tensor& out, torch::Tensor& exp_sums,
730 - torch::Tensor& max_logits, torch::Tensor& tmp_out,
731 - torch::Tensor& query, torch::Tensor& key_cache,
732 - torch::Tensor& value_cache, int num_kv_heads,
733 - float scale, torch::Tensor& block_tables,
734 - torch::Tensor& seq_lens, int block_size,
735 - int max_seq_len,
736 - const c10::optional<torch::Tensor>& alibi_slopes,
737 - const std::string& kv_cache_dtype, float kv_scale) {
732 +void paged_attention_v2(
733 + torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
734 + torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
735 + torch::Tensor& value_cache, int num_kv_heads, float scale,
736 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
737 + int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
738 + const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
739 + const int blocksparse_local_blocks, const int blocksparse_vert_stride,
740 + const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
741 TORCH_CHECK(kv_scale == 1.0f);
742 + TORCH_CHECK(blocksparse_vert_stride <= 1,
743 + "CPU backend does not support blocksparse attention yet.");
744 VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v2_impl",
745 [&] {
746 CPU_KERNEL_GUARD_IN(paged_attention_v2_impl)
csrc/ops.h
+18 -17
@@ -2,23 +2,24 @@
2
3 #include <torch/extension.h>
4
5 -void paged_attention_v1(torch::Tensor& out, torch::Tensor& query,
6 - torch::Tensor& key_cache, torch::Tensor& value_cache,
7 - int num_kv_heads, float scale,
8 - torch::Tensor& block_tables, torch::Tensor& seq_lens,
9 - int block_size, int max_seq_len,
10 - const c10::optional<torch::Tensor>& alibi_slopes,
11 - const std::string& kv_cache_dtype, float kv_scale);
12 -
13 -void paged_attention_v2(torch::Tensor& out, torch::Tensor& exp_sums,
14 - torch::Tensor& max_logits, torch::Tensor& tmp_out,
15 - torch::Tensor& query, torch::Tensor& key_cache,
16 - torch::Tensor& value_cache, int num_kv_heads,
17 - float scale, torch::Tensor& block_tables,
18 - torch::Tensor& seq_lens, int block_size,
19 - int max_seq_len,
20 - const c10::optional<torch::Tensor>& alibi_slopes,
21 - const std::string& kv_cache_dtype, float kv_scale);
5 +void paged_attention_v1(
6 + torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
7 + torch::Tensor& value_cache, int num_kv_heads, float scale,
8 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
9 + int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
10 + const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
11 + const int blocksparse_local_blocks, const int blocksparse_vert_stride,
12 + const int blocksparse_block_size, const int blocksparse_head_sliding_step);
13 +
14 +void paged_attention_v2(
15 + torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
16 + torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
17 + torch::Tensor& value_cache, int num_kv_heads, float scale,
18 + torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
19 + int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
20 + const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
21 + const int blocksparse_local_blocks, const int blocksparse_vert_stride,
22 + const int blocksparse_block_size, const int blocksparse_head_sliding_step);
23
24 void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
25 float epsilon);
docs/source/models/supported_models.rst
+4
@@ -123,6 +123,10 @@ Alongside each architecture, we include some popular models that use it.
123 - Phi-3
124 - :code:`microsoft/Phi-3-mini-4k-instruct`, :code:`microsoft/Phi-3-mini-128k-instruct`, etc.
125 -
126 + * - :code:`Phi3SmallForCausalLM`
127 + - Phi-3-Small
128 + - :code:`microsoft/Phi-3-small-8k-instruct`, :code:`microsoft/Phi-3-small-128k-instruct`, etc.
129 + -
130 * - :code:`QWenLMHeadModel`
131 - Qwen
132 - :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc.
tests/kernels/test_blocksparse_attention.py new
+442
@@ -0,0 +1,442 @@
1 +import random
2 +from typing import List, Optional, Tuple
3 +
4 +import pytest
5 +import torch
6 +
7 +from vllm import _custom_ops as ops
8 +from vllm.attention.ops.blocksparse_attention.interface import (
9 + LocalStridedBlockSparseAttn)
10 +from vllm.utils import get_max_shared_memory_bytes, is_hip
11 +
12 +from .allclose_default import get_default_atol, get_default_rtol
13 +
14 +FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
15 +# This will change depending on the compute capability.
16 +# - 512 as a buffer
17 +MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512
18 +# MAX_SEQ_LEN = 2771
19 +
20 +# There may not be enough gpu memory due to large NUM_BLOCKS.
21 +# Reduce NUM_BLOCKS when it happens.
22 +NUM_BLOCKS = 4321 # Arbitrary values for testing
23 +PARTITION_SIZE = 512
24 +DTYPES = [torch.half, torch.bfloat16]
25 +NUM_GEN_SEQS = [3] # Arbitrary values for testing
26 +NUM_PREFILL_SEQS = [3] # Arbitrary values for testing
27 +NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
28 +
29 +HEAD_SIZES = [64, 112]
30 +BLOCK_SIZES = [16, 32]
31 +USE_ALIBI = [False, True]
32 +KV_CACHE_DTYPE = ["auto", "fp8"]
33 +SEEDS = [0]
34 +CUDA_DEVICES = ['cuda:0']
35 +BLOCKSPARSE_LOCAL_BLOCKS = [16]
36 +BLOCKSPARSE_VERT_STRIDES = [8]
37 +
38 +BLOCKSPARSE_BLOCK_SIZES = [64]
39 +BLOCKSPARSE_HEADS_SLIDINGS = [0, 2, -1]
40 +BLOCKSPARSE_HOMO_HEADS = [True, False]
41 +
42 +
43 +def ref_masked_attention(
44 + query: torch.Tensor,
45 + key: torch.Tensor,
46 + value: torch.Tensor,
47 + scale: float,
48 + attn_mask: Optional[torch.Tensor] = None,
49 +) -> torch.Tensor:
50 + attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
51 + if attn_mask is not None:
52 + attn_weights = attn_weights + attn_mask.float()
53 + attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
54 + out = torch.einsum("hqk,khd->qhd", attn_weights, value)
55 + return out
56 +
57 +
58 +def ref_single_query_cached_kv_attention(
59 + output: torch.Tensor,
60 + query: torch.Tensor,
61 + num_queries_per_kv: int,
62 + key_cache: torch.Tensor,
63 + value_cache: torch.Tensor,
64 + block_tables: torch.Tensor,
65 + seq_lens: torch.Tensor,
66 + scale: float,
67 + alibi_slopes: Optional[torch.Tensor],
68 + tp_rank: int = 0,
69 + blocksparse_local_blocks: int = 0,
70 + blocksparse_vert_stride: int = 1,
71 + blocksparse_block_size: int = 64,
72 + blocksparse_head_sliding_step: int = 0,
73 +) -> None:
74 + num_query_heads = query.shape[1]
75 + num_kv_heads = value_cache.shape[1]
76 + head_size = value_cache.shape[2]
77 + block_size = value_cache.shape[3]
78 + num_seqs = query.shape[0]
79 +
80 + block_tables = block_tables.cpu().tolist()
81 + seq_lens = seq_lens.cpu().tolist()
82 + for i in range(num_seqs):
83 + q = query[i].unsqueeze(0)
84 + block_table = block_tables[i]
85 + seq_len = int(seq_lens[i])
86 +
87 + keys = []
88 + values = []
89 + for j in range(seq_len):
90 + block_number = int(block_table[j // block_size])
91 + block_offset = j % block_size
92 +
93 + k = key_cache[block_number, :, :, block_offset, :]
94 + k = k.reshape(num_kv_heads, head_size)
95 + keys.append(k)
96 +
97 + v = value_cache[block_number, :, :, block_offset]
98 + values.append(v)
99 + keys = torch.stack(keys, dim=0)
100 + values = torch.stack(values, dim=0)
101 + if num_queries_per_kv > 1:
102 + # Handle MQA and GQA
103 + keys = torch.repeat_interleave(keys, num_queries_per_kv, dim=1)
104 + values = torch.repeat_interleave(values, num_queries_per_kv, dim=1)
105 +
106 + alibi_bias = None
107 + if alibi_slopes is not None:
108 + # Create the ALiBi bias used in the paged attention kernel.
109 + position_ids = torch.arange(seq_len).int()
110 + alibi_bias = (position_ids - seq_len + 1).float()
111 + alibi_bias = alibi_slopes.view(-1, 1, 1) * alibi_bias.view(
112 + 1, 1, -1)
113 +
114 + if blocksparse_vert_stride >= 1:
115 + bsize = blocksparse_block_size
116 + hsliding = blocksparse_head_sliding_step
117 + vert = blocksparse_vert_stride
118 + locals = blocksparse_local_blocks
119 + qb = (seq_len - 1) // bsize
120 + attn_mask = q.new_zeros(
121 + (num_query_heads, 1, seq_len)).float() - torch.inf
122 + for h in range(num_query_heads):
123 + if hsliding >= 0: # slide with q heads
124 + bs_offset = (tp_rank * num_query_heads + h) * hsliding + 1
125 + else: # slide with kv heads
126 + bs_offset = (tp_rank * num_kv_heads +
127 + h // num_queries_per_kv) * (-hsliding) + 1
128 + for kb in range(qb + 1):
129 + kj = kb * bsize
130 + if (qb - kb) < locals or \
131 + (kb + bs_offset) % vert == 0:
132 + attn_mask[h, 0, kj:min(kj + bsize, seq_len)] = 0
133 + if alibi_bias is not None:
134 + attn_mask += alibi_bias
135 + else:
136 + attn_mask = alibi_bias
137 +
138 + out = ref_masked_attention(q, keys, values, scale, attn_mask=attn_mask)
139 + out = out.view(num_query_heads, head_size)
140 + output[i].copy_(out, non_blocking=True)
141 +
142 +
143 +@pytest.mark.parametrize("version", ["v1", "v2"])
144 +@pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
145 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
146 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
147 +@pytest.mark.parametrize("use_alibi", USE_ALIBI)
148 +@pytest.mark.parametrize("block_size", BLOCK_SIZES)
149 +@pytest.mark.parametrize("dtype", DTYPES)
150 +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
151 +@pytest.mark.parametrize("seed", SEEDS)
152 +@pytest.mark.parametrize("device", CUDA_DEVICES)
153 +@pytest.mark.parametrize("blocksparse_local_blocks", BLOCKSPARSE_LOCAL_BLOCKS)
154 +@pytest.mark.parametrize("blocksparse_vert_stride", BLOCKSPARSE_VERT_STRIDES)
155 +@pytest.mark.parametrize("blocksparse_block_size", BLOCKSPARSE_BLOCK_SIZES)
156 +@pytest.mark.parametrize("blocksparse_head_sliding_step",
157 + BLOCKSPARSE_HEADS_SLIDINGS)
158 +def test_paged_attention(
159 + kv_cache_factory,
160 + version: str,
161 + num_seqs: int,
162 + num_heads: Tuple[int, int],
163 + head_size: int,
164 + use_alibi: bool,
165 + block_size: int,
166 + dtype: torch.dtype,
167 + kv_cache_dtype: str,
168 + seed: int,
169 + device: str,
170 + blocksparse_local_blocks: int,
171 + blocksparse_vert_stride: int,
172 + blocksparse_block_size: int,
173 + blocksparse_head_sliding_step: int,
174 +) -> None:
175 + random.seed(seed)
176 + torch.random.manual_seed(seed)
177 + if torch.cuda.is_available():
178 + torch.cuda.manual_seed(seed)
179 + torch.set_default_device(device)
180 + scale = float(1.0 / (head_size**0.5))
181 + num_query_heads, num_kv_heads = num_heads
182 + query = torch.empty(num_seqs, num_query_heads, head_size, dtype=dtype)
183 + query.uniform_(-scale, scale)
184 +
185 + assert num_query_heads % num_kv_heads == 0
186 + num_queries_per_kv = num_query_heads // num_kv_heads
187 + alibi_slopes = None
188 + if use_alibi:
189 + alibi_slopes = torch.rand(num_query_heads, dtype=torch.float)
190 +
191 + seq_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
192 + seq_lens[-1] = MAX_SEQ_LEN
193 + max_seq_len = max(seq_lens)
194 + seq_lens = torch.tensor(seq_lens, dtype=torch.int)
195 +
196 + # Create the block tables.
197 + max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
198 + block_tables = []
199 + for _ in range(num_seqs):
200 + block_table = [
201 + random.randint(0, NUM_BLOCKS - 1)
202 + for _ in range(max_num_blocks_per_seq)
203 + ]
204 + block_tables.append(block_table)
205 + block_tables = torch.tensor(block_tables, dtype=torch.int)
206 +
207 + # Create the KV caches.
208 + key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
209 + num_kv_heads, head_size,
210 + kv_cache_dtype, dtype, seed,
211 + device)
212 + key_cache, value_cache = key_caches[0], value_caches[0]
213 +
214 + # Using default kv_scale
215 + kv_scale = 1.0
216 + tp_rank = 0
217 +
218 + # Call the paged attention kernel.
219 + output = torch.empty_like(query)
220 + if version == "v1":
221 + ops.paged_attention_v1(
222 + output,
223 + query,
224 + key_cache,
225 + value_cache,
226 + num_kv_heads,
227 + scale,
228 + block_tables,
229 + seq_lens,
230 + block_size,
231 + max_seq_len,
232 + alibi_slopes,
233 + kv_cache_dtype,
234 + kv_scale,
235 + tp_rank=tp_rank,
236 + blocksparse_local_blocks=blocksparse_local_blocks,
237 + blocksparse_vert_stride=blocksparse_vert_stride,
238 + blocksparse_block_size=blocksparse_block_size,
239 + blocksparse_head_sliding_step=blocksparse_head_sliding_step,
240 + )
241 + elif version == "v2":
242 + num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
243 + assert PARTITION_SIZE % block_size == 0
244 + num_seqs, num_heads, head_size = output.shape
245 + tmp_output = torch.empty(
246 + size=(num_seqs, num_heads, num_partitions, head_size),
247 + dtype=output.dtype,
248 + )
249 + exp_sums = torch.empty(
250 + size=(num_seqs, num_heads, num_partitions),
251 + dtype=torch.float32,
252 + )
253 + max_logits = torch.empty_like(exp_sums)
254 + ops.paged_attention_v2(
255 + output,
256 + exp_sums,
257 + max_logits,
258 + tmp_output,
259 + query,
260 + key_cache,
261 + value_cache,
262 + num_kv_heads,
263 + scale,
264 + block_tables,
265 + seq_lens,
266 + block_size,
267 + max_seq_len,
268 + alibi_slopes,
269 + kv_cache_dtype,
270 + kv_scale,
271 + tp_rank=tp_rank,
272 + blocksparse_local_blocks=blocksparse_local_blocks,
273 + blocksparse_vert_stride=blocksparse_vert_stride,
274 + blocksparse_block_size=blocksparse_block_size,
275 + blocksparse_head_sliding_step=blocksparse_head_sliding_step,
276 + )
277 + else:
278 + raise AssertionError(f"Unknown version: {version}")
279 +
280 + # Run the reference implementation.
281 + if kv_cache_dtype == "fp8":
282 + # Convert cache data back to dtype.
283 + x = 16 // torch.tensor([], dtype=dtype).element_size()
284 + key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x,
285 + block_size, x)
286 + dequantized_key_cache = torch.empty(size=key_cache_shape,
287 + dtype=dtype,
288 + device=device)
289 + ops.convert_fp8(dequantized_key_cache, key_cache)
290 + key_cache = dequantized_key_cache
291 +
292 + value_cache_shape = value_cache.shape
293 + dequantized_value_cache = torch.empty(size=value_cache_shape,
294 + dtype=dtype,
295 + device=device)
296 + ops.convert_fp8(dequantized_value_cache, value_cache)
297 + value_cache = dequantized_value_cache
298 +
299 + ref_output = torch.empty_like(query)
300 + ref_single_query_cached_kv_attention(
301 + ref_output,
302 + query,
303 + num_queries_per_kv,
304 + key_cache,
305 + value_cache,
306 + block_tables,
307 + seq_lens,
308 + scale,
309 + alibi_slopes,
310 + tp_rank,
311 + blocksparse_local_blocks,
312 + blocksparse_vert_stride,
313 + blocksparse_block_size,
314 + blocksparse_head_sliding_step,
315 + )
316 +
317 + # NOTE(woosuk): Due to the kernel-level differences in the two
318 + # implementations, there is a small numerical difference in the two
319 + # outputs. Thus, we use a relaxed tolerance for the test.
320 + atol = get_default_atol(output) if is_hip() else 1e-3
321 + rtol = get_default_rtol(output) if is_hip() else 1e-5
322 +
323 + # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error,
324 + # so we use a relaxed tolerance for the test.
325 + atol, rtol = 1e-3, 1e-5
326 + if kv_cache_dtype == "fp8":
327 + atol, rtol = 1e-2, 1e-5
328 + assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
329 +
330 +
331 +def ref_multi_query_kv_attention(
332 + cu_seq_lens: List[int],
333 + query: torch.Tensor,
334 + key: torch.Tensor,
335 + value: torch.Tensor,
336 + scale: float,
337 + dtype: torch.dtype,
338 +) -> torch.Tensor:
339 + num_seqs = len(cu_seq_lens) - 1
340 + ref_outputs = []
341 + for i in range(num_seqs):
342 + start_idx = cu_seq_lens[i]
343 + end_idx = cu_seq_lens[i + 1]
344 + seq_len = end_idx - start_idx
345 +
346 + # Create attention mask.
347 + attn_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=dtype),
348 + diagonal=1)
349 + attn_mask = attn_mask * torch.finfo(dtype).min
350 + attn_mask = attn_mask.to(dtype=dtype)
351 +
352 + ref_output = ref_masked_attention(
353 + query[start_idx:end_idx],
354 + key[start_idx:end_idx],
355 + value[start_idx:end_idx],
356 + scale,
357 + attn_mask=attn_mask,
358 + )
359 + ref_outputs.append(ref_output)
360 + ref_output = torch.cat(ref_outputs, dim=0)
361 + return ref_output
362 +
363 +
364 +@pytest.mark.parametrize("num_seqs", NUM_PREFILL_SEQS)
365 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
366 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
367 +@pytest.mark.parametrize("blocksparse_local_blocks", BLOCKSPARSE_LOCAL_BLOCKS)
368 +@pytest.mark.parametrize("blocksparse_vert_stride", BLOCKSPARSE_VERT_STRIDES)
369 +@pytest.mark.parametrize("blocksparse_block_size", BLOCKSPARSE_BLOCK_SIZES)
370 +@pytest.mark.parametrize("blocksparse_homo_heads", BLOCKSPARSE_HOMO_HEADS)
371 +@pytest.mark.parametrize("dtype", DTYPES)
372 +@pytest.mark.parametrize("seed", SEEDS)
373 +@pytest.mark.parametrize("device", CUDA_DEVICES)
374 +@torch.inference_mode()
375 +def test_varlen_blocksparse_attention_prefill(
376 + num_seqs: int,
377 + num_heads: Tuple[int, int],
378 + head_size: int,
379 + blocksparse_local_blocks: int,
380 + blocksparse_vert_stride: int,
381 + blocksparse_block_size: int,
382 + blocksparse_homo_heads: bool,
383 + dtype: torch.dtype,
384 + seed: int,
385 + device: str,
386 +) -> None:
387 + random.seed(seed)
388 + torch.random.manual_seed(seed)
389 + if torch.cuda.is_available():
390 + torch.cuda.manual_seed(seed)
391 + torch.set_default_device(device)
392 + # MAX_SEQ_LEN sometimes causes OOM in the reference implementation.
393 + # As the xformers library is already tested with its own tests, we can use
394 + # a smaller MAX_SEQ_LEN here.
395 + max_len = min(MAX_SEQ_LEN, 4096)
396 + seq_lens = random.sample(range(1, max_len), num_seqs)
397 + cu_seq_lens = torch.cumsum(torch.tensor([0] + seq_lens), dim=0)
398 + num_tokens = sum(seq_lens)
399 +
400 + scale = float(1.0 / (head_size**0.5))
401 + num_query_heads, num_kv_heads = num_heads
402 + assert num_query_heads % num_kv_heads == 0
403 + num_queries_per_kv = num_query_heads // num_kv_heads
404 +
405 + qkv = torch.empty(num_tokens,
406 + num_query_heads + 2 * num_kv_heads,
407 + head_size,
408 + dtype=dtype)
409 + qkv.uniform_(-scale, scale)
410 + query, key, value = qkv.split(
411 + [num_query_heads, num_kv_heads, num_kv_heads], dim=1)
412 +
413 + bs_attn_op = LocalStridedBlockSparseAttn(
414 + num_query_heads,
415 + max_len,
416 + local_blocks=blocksparse_local_blocks,
417 + vert_stride=blocksparse_vert_stride,
418 + block_size=blocksparse_block_size,
419 + device=device,
420 + dtype=dtype,
421 + homo_head=blocksparse_homo_heads)
422 +
423 + output = bs_attn_op(query,
424 + key,
425 + value,
426 + cu_seq_lens.to(device),
427 + sm_scale=scale)
428 +
429 + if num_queries_per_kv > 1:
430 + # Handle MQA and GQA
431 + key = torch.repeat_interleave(key, num_queries_per_kv, dim=1)
432 + value = torch.repeat_interleave(value, num_queries_per_kv, dim=1)
433 +
434 + ref_output = ref_multi_query_kv_attention(
435 + cu_seq_lens,
436 + query,
437 + key,
438 + value,
439 + scale,
440 + dtype,
441 + )
442 + assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2)
vllm/_custom_ops.py
+21 -9
@@ -45,11 +45,17 @@ def paged_attention_v1(
45 alibi_slopes: Optional[torch.Tensor],
46 kv_cache_dtype: str,
47 kv_scale: float,
48 + tp_rank: int = 0,
49 + blocksparse_local_blocks: int = 0,
50 + blocksparse_vert_stride: int = 0,
51 + blocksparse_block_size: int = 64,
52 + blocksparse_head_sliding_step: int = 0,
53 ) -> None:
49 - vllm_ops.paged_attention_v1(out, query, key_cache, value_cache,
50 - num_kv_heads, scale, block_tables, seq_lens,
51 - block_size, max_seq_len, alibi_slopes,
52 - kv_cache_dtype, kv_scale)
54 + vllm_ops.paged_attention_v1(
55 + out, query, key_cache, value_cache, num_kv_heads, scale, block_tables,
56 + seq_lens, block_size, max_seq_len, alibi_slopes, kv_cache_dtype,
57 + kv_scale, tp_rank, blocksparse_local_blocks, blocksparse_vert_stride,
58 + blocksparse_block_size, blocksparse_head_sliding_step)
59
60
61 def paged_attention_v2(
@@ -69,12 +75,18 @@ def paged_attention_v2(
75 alibi_slopes: Optional[torch.Tensor],
76 kv_cache_dtype: str,
77 kv_scale: float,
78 + tp_rank: int = 0,
79 + blocksparse_local_blocks: int = 0,
80 + blocksparse_vert_stride: int = 0,
81 + blocksparse_block_size: int = 64,
82 + blocksparse_head_sliding_step: int = 0,
83 ) -> None:
73 - vllm_ops.paged_attention_v2(out, exp_sum, max_logits, tmp_out, query,
74 - key_cache, value_cache, num_kv_heads, scale,
75 - block_tables, seq_lens, block_size,
76 - max_seq_len, alibi_slopes, kv_cache_dtype,
77 - kv_scale)
84 + vllm_ops.paged_attention_v2(
85 + out, exp_sum, max_logits, tmp_out, query, key_cache, value_cache,
86 + num_kv_heads, scale, block_tables, seq_lens, block_size, max_seq_len,
87 + alibi_slopes, kv_cache_dtype, kv_scale, tp_rank,
88 + blocksparse_local_blocks, blocksparse_vert_stride,
89 + blocksparse_block_size, blocksparse_head_sliding_step)
90
91
92 # pos encoding ops
vllm/attention/backends/abstract.py
+1
@@ -111,6 +111,7 @@ class AttentionImpl(ABC, Generic[T]):
111 alibi_slopes: Optional[List[float]] = None,
112 sliding_window: Optional[int] = None,
113 kv_cache_dtype: str = "auto",
114 + blocksparse_params: Optional[Dict[str, Any]] = None,
115 ) -> None:
116 raise NotImplementedError
117
vllm/attention/backends/blocksparse_attn.py new
+410
@@ -0,0 +1,410 @@
1 +from dataclasses import dataclass, field
2 +from typing import Any, Dict, List, Optional, Tuple, Type
3 +
4 +import torch
5 +
6 +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
7 + AttentionMetadata)
8 +from vllm.attention.ops.blocksparse_attention.interface import (
9 + LocalStridedBlockSparseAttn, get_head_sliding_step)
10 +from vllm.attention.ops.paged_attn import PagedAttention
11 +from vllm.distributed import (get_tensor_model_parallel_rank,
12 + get_tensor_model_parallel_world_size)
13 +
14 +
15 +@dataclass
16 +class BlocksparseParams:
17 + max_seqlen: int
18 +
19 + # Num q heads per tensor-parallel rank/partition
20 + num_heads: int # per TP partition
21 + # Num kv heads per tensor-parallel rank/partition
22 + num_kv_heads: int
23 +
24 + # block size used for blocksparse attention.
25 + # This is the block_size used in `local_blocks`, `vert_stride`.
26 + block_size: int
27 +
28 + # Number of blocks for local attention, i.e., number of
29 + # local attended tokens / `sparse_block_size`
30 + local_blocks: int
31 +
32 + # Attend to one block per every `vert_stride` blocks.
33 + # Controlling the sparsity
34 + vert_stride: int
35 + """
36 + If to use the same vertical stride offset for all heads,
37 + i.e., attend to the same block of tokens on all heads.
38 + By default, it is False, i.e., attention on the non-local
39 + blocks depends on the `head_idx`, that is on
40 + blocks satisfying
41 + `(block_idx + head_idx * head_sliding_step + 1) % vert_stride == 0`
42 + where `head_sliding_step=max(1, int(vert_stride / num_total_heads))`,
43 + `block_idx = position_id // sparse_block_size`.
44 + See `..ops.blocksparse_attention.utils:get_sparse_attn_mask`
45 + for more detail.
46 + """
47 + homo_head: bool = False
48 +
49 + # If within a group, the kv offsets that each q attends is the same or no.
50 + homo_head_group: bool = False
51 +
52 + # Decided by homo_head and homo_head group
53 + head_sliding_step: int = field(init=False)
54 +
55 + # range of q heads to for a TP rank
56 + active_head_range: Tuple = field(init=False)
57 +
58 + def __post_init__(self):
59 + assert self.block_size > 0
60 + assert self.local_blocks >= 0
61 + assert self.vert_stride >= 1
62 + assert self.num_heads % self.num_kv_heads == 0
63 +
64 + tp_size = get_tensor_model_parallel_world_size()
65 + tp_rank = get_tensor_model_parallel_rank()
66 + total_heads = tp_size * self.num_heads
67 + total_kv_heads = tp_size * self.num_kv_heads
68 +
69 + if self.homo_head:
70 + self.head_sliding_step = 0
71 + elif self.homo_head_group:
72 + head_sliding_step = get_head_sliding_step(total_kv_heads,
73 + self.vert_stride)
74 + # negative indicates sliding along kv heads, i.e., homo q group
75 + self.head_sliding_step = -head_sliding_step
76 + else:
77 + self.head_sliding_step = get_head_sliding_step(
78 + total_heads, self.vert_stride)
79 +
80 + self.active_head_range = (
81 + tp_rank * self.num_heads,
82 + (tp_rank + 1) * self.num_heads,
83 + )
84 +
85 +
86 +class BlocksparseFlashAttentionBackend(AttentionBackend):
87 +
88 + @staticmethod
89 + def get_impl_cls() -> Type["BlocksparseFlashAttentionImpl"]:
90 + return BlocksparseFlashAttentionImpl
91 +
92 + @staticmethod
93 + def make_metadata(*args, **kwargs) -> "BlocksparseFlashAttentionMetadata":
94 + return BlocksparseFlashAttentionMetadata(*args, **kwargs)
95 +
96 + @staticmethod
97 + def get_kv_cache_shape(
98 + num_blocks: int,
99 + block_size: int,
100 + num_kv_heads: int,
101 + head_size: int,
102 + ) -> Tuple[int, ...]:
103 + return PagedAttention.get_kv_cache_shape(num_blocks, block_size,
104 + num_kv_heads, head_size)
105 +
106 + @staticmethod
107 + def swap_blocks(
108 + src_kv_cache: torch.Tensor,
109 + dst_kv_cache: torch.Tensor,
110 + src_to_dst: Dict[int, int],
111 + ) -> None:
112 + PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
113 +
114 + @staticmethod
115 + def copy_blocks(
116 + kv_caches: List[torch.Tensor],
117 + src_to_dists: Dict[int, List[int]],
118 + ) -> None:
119 + PagedAttention.copy_blocks(kv_caches, src_to_dists)
120 +
121 +
122 +@dataclass
123 +class BlocksparseFlashAttentionMetadata(AttentionMetadata):
124 + """A copy of Metadata for FlashAttentionBackend,
125 + to avoid having to install flash_attn.
126 +
127 + NOTE: Any python object stored here is not updated when it is
128 + cuda-graph replayed. If you have values that need to be changed
129 + dynamically, it should be stored in tensor. The tensor has to be
130 + updated from `CUDAGraphRunner.forward` API.
131 + """
132 + # (batch_size,). The sequence length per sequence. Sequence length means
133 + # the computed tokens + new tokens None if it is a decoding.
134 + seq_lens: Optional[List[int]]
135 + # seq_lens stored as a tensor.
136 + seq_lens_tensor: Optional[torch.Tensor]
137 +
138 + # NOTE(sang): Definition of context_len, query_len, and seq_len.
139 + # |---------- N-1 iteration --------|
140 + # |---------------- N iteration ---------------------|
141 + # |- tokenA -|......................|-- newTokens ---|
142 + # |---------- context_len ----------|
143 + # |-------------------- seq_len ----------------------|
144 + # |-- query_len ---|
145 +
146 + # Maximum query length in the batch. None for decoding.
147 + max_query_len: Optional[int]
148 + # Maximum sequence length among prefill batch. 0 if there are decoding
149 + # requests only.
150 + max_prefill_seq_len: int
151 + # Maximum sequence length among decode batch. 0 if there are prefill
152 + # requests only.
153 + max_decode_seq_len: int
154 + # (batch_size + 1,). The cumulative subquery lengths of the sequences in
155 + # the batch, used to index into subquery. E.g., if the subquery length
156 + # is [4, 6], it is [0, 4, 10].
157 + query_start_loc: Optional[torch.Tensor]
158 + # (batch_size + 1,). The cumulative sequence lengths of the sequences in
159 + # the batch, used to index into sequence. E.g., if the sequence length is
160 + # [4, 6], it is [0, 4, 10].
161 + seq_start_loc: Optional[torch.Tensor]
162 + # (batch_size,) A tensor of context lengths (tokens that are computed
163 + # so far).
164 + context_lens_tensor: Optional[torch.Tensor]
165 +
166 + # (batch_size, max_blocks_per_seq).
167 + # Block addresses per sequence. (Seq id -> list of physical block)
168 + # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
169 + # in the kv cache. Each block can contain up to block_size tokens.
170 + # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
171 + # captured.
172 + block_tables: Optional[torch.Tensor]
173 +
174 + # Whether or not if cuda graph is enabled.
175 + # Cuda-graph is currently enabled for decoding only.
176 + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
177 + use_cuda_graph: bool
178 +
179 + _cached_prefill_metadata: Optional[
180 + "BlocksparseFlashAttentionMetadata"] = None
181 + _cached_decode_metadata: Optional[
182 + "BlocksparseFlashAttentionMetadata"] = None
183 +
184 + @property
185 + def prefill_metadata(
186 + self) -> Optional["BlocksparseFlashAttentionMetadata"]:
187 + if self.num_prefills == 0:
188 + return None
189 +
190 + if self._cached_prefill_metadata is not None:
191 + return self._cached_prefill_metadata
192 +
193 + assert self.seq_lens is not None
194 + assert self.seq_lens_tensor is not None
195 + assert self.query_start_loc is not None
196 + assert self.context_lens_tensor is not None
197 + assert self.block_tables is not None
198 + assert self.seq_start_loc is not None
199 +
200 + self._cached_prefill_metadata = BlocksparseFlashAttentionMetadata(
201 + num_prefills=self.num_prefills,
202 + num_prefill_tokens=self.num_prefill_tokens,
203 + num_decode_tokens=0,
204 + slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
205 + seq_lens=self.seq_lens[:self.num_prefills],
206 + seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
207 + max_query_len=self.max_query_len,
208 + max_prefill_seq_len=self.max_prefill_seq_len,
209 + max_decode_seq_len=0,
210 + query_start_loc=self.query_start_loc[:self.num_prefills + 1],
211 + seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
212 + context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
213 + block_tables=self.block_tables[:self.num_prefills],
214 + use_cuda_graph=False,
215 + )
216 + return self._cached_prefill_metadata
217 +
218 + @property
219 + def decode_metadata(self) -> Optional["BlocksparseFlashAttentionMetadata"]:
220 + if self.num_decode_tokens == 0:
221 + return None
222 +
223 + if self._cached_decode_metadata is not None:
224 + return self._cached_decode_metadata
225 + assert self.block_tables is not None
226 + assert self.seq_lens_tensor is not None
227 +
228 + self._cached_decode_metadata = BlocksparseFlashAttentionMetadata(
229 + num_prefills=0,
230 + num_prefill_tokens=0,
231 + num_decode_tokens=self.num_decode_tokens,
232 + slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
233 + seq_lens=None,
234 + seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
235 + max_query_len=None,
236 + max_prefill_seq_len=0,
237 + max_decode_seq_len=self.max_decode_seq_len,
238 + query_start_loc=None,
239 + seq_start_loc=None,
240 + context_lens_tensor=None,
241 + block_tables=self.block_tables[self.num_prefills:],
242 + use_cuda_graph=self.use_cuda_graph,
243 + )
244 + return self._cached_decode_metadata
245 +
246 +
247 +class BlocksparseFlashAttentionImpl(AttentionImpl):
248 + """
249 + If the input tensors contain prompt tokens, the layout is as follows:
250 + |<--------------- num_prompt_tokens -------------->|
251 + |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|
252 +
253 + Otherwise, the layout is as follows:
254 + |<------------------ num_generation_tokens (M) ----------------->|
255 + |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
256 +
257 + Generation tokens can contain padding when cuda-graph is used.
258 + Currently, prompt tokens don't contain any padding.
259 +
260 + The prompts might have different lengths, while the generation tokens
261 + always have length 1.
262 +
263 + """
264 +
265 + def __init__(
266 + self,
267 + num_heads: int,
268 + head_size: int,
269 + scale: float,
270 + num_kv_heads: int,
271 + alibi_slopes: Optional[List[float]],
272 + sliding_window: Optional[int],
273 + kv_cache_dtype: str,
274 + blocksparse_params: Optional[Dict[str, Any]] = None,
275 + ) -> None:
276 + assert blocksparse_params is not None
277 + assert alibi_slopes is None, ValueError(
278 + "Alibi not support for blocksparse flash attention.")
279 + assert sliding_window is None, ValueError(
280 + "sliding_window is invalid for blocksparse attention.")
281 +
282 + if "num_heads" not in blocksparse_params:
283 + blocksparse_params["num_heads"] = num_heads
284 + if "num_kv_heads" not in blocksparse_params:
285 + blocksparse_params["num_kv_heads"] = num_kv_heads or num_heads
286 + self.blocksparse_params = BlocksparseParams(**blocksparse_params)
287 + self.kv_cache_dtype = kv_cache_dtype
288 +
289 + self.num_heads = num_heads
290 + self.head_size = head_size
291 + self.scale = float(scale)
292 + self.alibi_slopes = alibi_slopes
293 + self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
294 +
295 + assert self.num_heads % self.num_kv_heads == 0
296 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads
297 +
298 + self.local_blocks = self.blocksparse_params.local_blocks
299 + self.vert_stride = self.blocksparse_params.vert_stride
300 + self.sparse_block_size = self.blocksparse_params.block_size
301 + self.head_sliding_step = self.blocksparse_params.head_sliding_step
302 +
303 + suppored_head_sizes = PagedAttention.get_supported_head_sizes()
304 + if head_size not in suppored_head_sizes:
305 + raise ValueError(
306 + f"Head size {head_size} is not supported by PagedAttention. "
307 + f"Supported head sizes are: {suppored_head_sizes}.")
308 +
309 + self.tp_size = get_tensor_model_parallel_world_size()
310 + self.tp_rank = get_tensor_model_parallel_rank()
311 +
312 + total_num_heads = num_heads * self.tp_size
313 + self.bs_attn = LocalStridedBlockSparseAttn(
314 + total_num_heads,
315 + self.blocksparse_params.max_seqlen,
316 + self.blocksparse_params.local_blocks,
317 + self.blocksparse_params.vert_stride,
318 + self.blocksparse_params.block_size,
319 + homo_head=self.blocksparse_params.homo_head,
320 + active_head_range=self.blocksparse_params.active_head_range,
321 + )
322 +
323 + def forward(
324 + self,
325 + query: torch.Tensor,
326 + key: torch.Tensor,
327 + value: torch.Tensor,
328 + kv_cache: torch.Tensor,
329 + attn_metadata: BlocksparseFlashAttentionMetadata,
330 + kv_scale: float = 1.0,
331 + ) -> torch.Tensor:
332 + """Forward pass with FlashAttention and PagedAttention.
333 +
334 + Args:
335 + query: shape = [num_tokens, num_heads * head_size]
336 + key: shape = [num_tokens, num_kv_heads * head_size]
337 + value: shape = [num_tokens, num_kv_heads * head_size]
338 + kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
339 + attn_metadata: Metadata for attention.
340 + Returns:
341 + shape = [num_tokens, num_heads * head_size]
342 + """
343 + num_tokens, hidden_size = query.shape
344 + # Reshape the query, key, and value tensors.
345 + query = query.view(-1, self.num_heads, self.head_size)
346 + key = key.view(-1, self.num_kv_heads, self.head_size)
347 + value = value.view(-1, self.num_kv_heads, self.head_size)
348 +
349 + if kv_cache is not None:
350 + key_cache, value_cache = PagedAttention.split_kv_cache(
351 + kv_cache, self.num_kv_heads, self.head_size)
352 +
353 + # Reshape the input keys and values and store them in the cache.
354 + # If kv_cache is not provided, the new key and value tensors are
355 + # not cached. This happens during the initial memory profiling run.
356 +
357 + PagedAttention.write_to_paged_cache(
358 + key,
359 + value,
360 + key_cache,
361 + value_cache,
362 + attn_metadata.slot_mapping,
363 + self.kv_cache_dtype,
364 + kv_scale,
365 + )
366 +
367 + if prefill_meta := attn_metadata.prefill_metadata:
368 +
369 + # Prompt run.
370 + # normal attention
371 + # When block_tables are not filled, it means q and k are the
372 + # prompt, and they have the same length.
373 +
374 + assert kv_cache is None \
375 + or prefill_meta.block_tables is None \
376 + or prefill_meta.block_tables.numel() == 0, \
377 + "Does not support prefix-enabled attention."
378 +
379 + output = self.bs_attn(
380 + q=query,
381 + k=key,
382 + v=value,
383 + cu_seqlens_q=prefill_meta.seq_start_loc,
384 + cu_seqlens_k=prefill_meta.seq_start_loc,
385 + sm_scale=self.scale,
386 + )
387 +
388 + if decode_meta := attn_metadata.decode_metadata:
389 + # Decoding run.
390 + output = PagedAttention.forward_decode(
391 + query,
392 + key_cache,
393 + value_cache,
394 + decode_meta.block_tables,
395 + decode_meta.seq_lens_tensor,
396 + self.blocksparse_params.max_seqlen,
397 + self.kv_cache_dtype,
398 + self.num_kv_heads,
399 + self.scale,
400 + self.alibi_slopes,
401 + kv_scale,
402 + tp_rank=self.tp_rank,
403 + blocksparse_local_blocks=self.local_blocks,
404 + blocksparse_vert_stride=self.vert_stride,
405 + blocksparse_block_size=self.sparse_block_size,
406 + blocksparse_head_sliding_step=self.head_sliding_step,
407 + )
408 +
409 + # Reshape the output tensor.
410 + return output.view(num_tokens, hidden_size)
vllm/attention/backends/flash_attn.py
+4 -1
@@ -1,6 +1,6 @@
1 """Attention layer with FlashAttention."""
2 from dataclasses import dataclass
3 -from typing import List, Optional, Tuple, Type
3 +from typing import Any, Dict, List, Optional, Tuple, Type
4
5 import torch
6 from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
@@ -219,7 +219,10 @@ class FlashAttentionImpl(AttentionImpl):
219 alibi_slopes: Optional[List[float]],
220 sliding_window: Optional[int],
221 kv_cache_dtype: str,
222 + blocksparse_params: Optional[Dict[str, Any]] = None,
223 ) -> None:
224 + assert blocksparse_params is None, ValueError(
225 + "FlashAttention does not support block-sparse attention.")
226 self.num_heads = num_heads
227 self.head_size = head_size
228 self.scale = float(scale)
vllm/attention/backends/rocm_flash_attn.py
+4 -1
@@ -1,6 +1,6 @@
1 """Attention layer ROCm GPUs."""
2 from dataclasses import dataclass
3 -from typing import List, Optional, Tuple, Type
3 +from typing import Any, Dict, List, Optional, Tuple, Type
4
5 import torch
6
@@ -201,7 +201,10 @@ class ROCmFlashAttentionImpl(AttentionImpl):
201 alibi_slopes: Optional[List[float]],
202 sliding_window: Optional[int],
203 kv_cache_dtype: str,
204 + blocksparse_params: Optional[Dict[str, Any]] = None,
205 ) -> None:
206 + assert blocksparse_params is None, ValueError(
207 + "ROCFlashAttention does not support blocksparse attention.")
208 self.num_heads = num_heads
209 self.head_size = head_size
210 self.scale = float(scale)
vllm/attention/backends/torch_sdpa.py
+4 -1
@@ -1,7 +1,7 @@
1 """ Attention layer with torch scaled_dot_product_attention
2 and PagedAttention."""
3 from dataclasses import dataclass
4 -from typing import List, Optional, Tuple, Type
4 +from typing import Any, Dict, List, Optional, Tuple, Type
5
6 import torch
7 from torch.nn.functional import scaled_dot_product_attention
@@ -100,7 +100,10 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
100 alibi_slopes: Optional[List[float]],
101 sliding_window: Optional[int],
102 kv_cache_dtype: str,
103 + blocksparse_params: Optional[Dict[str, Any]] = None,
104 ) -> None:
105 + assert blocksparse_params is None, ValueError(
106 + "Torch SPDA does not support block-sparse attention.")
107 self.num_heads = num_heads
108 self.head_size = head_size
109 self.scale = float(scale)
vllm/attention/backends/xformers.py
+4 -1
@@ -1,6 +1,6 @@
1 """Attention layer with xFormers and PagedAttention."""
2 from dataclasses import dataclass
3 -from typing import Dict, List, Optional, Tuple, Type
3 +from typing import Any, Dict, List, Optional, Tuple, Type
4
5 import torch
6 from xformers import ops as xops
@@ -212,7 +212,10 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
212 alibi_slopes: Optional[List[float]],
213 sliding_window: Optional[int],
214 kv_cache_dtype: str,
215 + blocksparse_params: Optional[Dict[str, Any]] = None,
216 ) -> None:
217 + assert blocksparse_params is None, ValueError(
218 + "XFormer does not support block-sparse attention.")
219 self.num_heads = num_heads
220 self.head_size = head_size
221 self.scale = float(scale)
vllm/attention/layer.py
+7 -3
@@ -1,5 +1,5 @@
1 """Attention layer."""
2 -from typing import List, Optional
2 +from typing import Any, Dict, List, Optional
3
4 import torch
5 import torch.nn as nn
@@ -33,6 +33,7 @@ class Attention(nn.Module):
33 sliding_window: Optional[int] = None,
34 cache_config: Optional[CacheConfig] = None,
35 quant_config: Optional[QuantizationConfig] = None,
36 + blocksparse_params: Optional[Dict[str, Any]] = None,
37 ) -> None:
38 super().__init__()
39 if cache_config is not None:
@@ -69,10 +70,12 @@ class Attention(nn.Module):
70 dtype = torch.get_default_dtype()
71 attn_backend = get_attn_backend(num_heads, head_size, num_kv_heads,
72 sliding_window, dtype, kv_cache_dtype,
72 - block_size)
73 + block_size, blocksparse_params
74 + is not None)
75 impl_cls = attn_backend.get_impl_cls()
76 self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
75 - alibi_slopes, sliding_window, kv_cache_dtype)
77 + alibi_slopes, sliding_window, kv_cache_dtype,
78 + blocksparse_params)
79
80 def forward(
81 self,
@@ -90,4 +93,5 @@ class Attention(nn.Module):
93 s += f", num_heads={self.impl.num_heads}" # type: ignore
94 s += f", num_kv_heads={self.impl.num_kv_heads}" # type: ignore
95 s += f", scale={self.impl.scale}" # type: ignore
96 + s += f", backend={self.impl.__class__.__name__}"
97 return s
vllm/attention/ops/blocksparse_attention/__init__.py
vllm/attention/ops/blocksparse_attention/blocksparse_attention_kernel.py new
+423
@@ -0,0 +1,423 @@
1 +import torch
2 +import triton
3 +import triton.language as tl
4 +
5 +
6 +def blocksparse_flash_attn_varlen_fwd(
7 + q,
8 + k,
9 + v, # (#tokens, n_heads, head_size)
10 + cu_seqlens_k,
11 + cu_seqlens_q,
12 + sm_scale,
13 + sparse_layout,
14 + *,
15 + block_size=64,
16 + q_block_size=None,
17 + max_seqlen=None):
18 + # split q to blocks
19 +
20 + assert isinstance(sparse_layout, (list, tuple))
21 +
22 + _, n_heads, head_size = q.shape
23 + batch_size = cu_seqlens_k.size(0) - 1
24 + q_block_size = q_block_size or block_size
25 +
26 + assert q.dim() == k.dim() == v.dim() == 3
27 + assert q.size(1) % k.size(1) == 0
28 + assert q.size(2) == k.size(2)
29 + # TODO(linxihui): allow k, v to have different head_size
30 + assert k.shape == v.shape
31 + assert cu_seqlens_k.dim() == 1
32 +
33 + q_k_ratio = q.size(1) // k.size(1)
34 +
35 + if cu_seqlens_q is None:
36 + if q.size(0) == batch_size: # decoding only
37 + cu_seqlens_q = torch.arange(
38 + 0,
39 + batch_size + 1,
40 + dtype=cu_seqlens_k.dtype,
41 + device=cu_seqlens_k.device,
42 + )
43 + elif q.size(0) == k.size(0):
44 + cu_seqlens_q = cu_seqlens_k
45 + else:
46 + raise ValueError("cu_seqlens_q must be specified\
47 + if it mix of prefilling and decoding.")
48 + else:
49 + assert cu_seqlens_k.size(0) == cu_seqlens_q.size(0)
50 +
51 + # switch to use cpu to avoid too many kernel launches when iterated over
52 + q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).cpu()
53 + k_lens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).cpu()
54 +
55 + assert torch.logical_or(q_lens == 1, k_lens == q_lens).all(), (
56 + "length of q should either be 1 (decoding) or same as k (prefilling).")
57 +
58 + if max_seqlen:
59 + assert k_lens.max() <= max_seqlen
60 +
61 + n_blocks = (q_lens + q_block_size - 1) // q_block_size
62 +
63 + q_batch_ids = torch.tensor(
64 + [i for i, n in enumerate(n_blocks) for _ in range(n)],
65 + dtype=cu_seqlens_q.dtype,
66 + device=cu_seqlens_q.device,
67 + )
68 + q_start_sids = torch.tensor(
69 + [i * q_block_size for n in n_blocks for i in range(n)],
70 + dtype=cu_seqlens_q.dtype,
71 + device=cu_seqlens_q.device,
72 + )
73 +
74 + out = q.new_empty(q.shape)
75 + cu_seqlens_q = cu_seqlens_q.contiguous()
76 + cu_seqlens_k = cu_seqlens_k.contiguous()
77 +
78 + layout_crow_indices, layout_col_indices = sparse_layout
79 + block_d = triton.next_power_of_2(head_size)
80 +
81 + decoding_only = (q_lens == 1).all().item()
82 + grid = (len(q_start_sids), n_heads, 1)
83 +
84 + _fwd_kernel_batch_inference[grid](
85 + q,
86 + k,
87 + v,
88 + out,
89 + sm_scale,
90 + cu_seqlens_q[:-1],
91 + cu_seqlens_q[1:],
92 + cu_seqlens_k[:-1],
93 + cu_seqlens_k[1:],
94 + q_batch_ids,
95 + q_start_sids,
96 + 0,
97 + *q.stride(),
98 + 0,
99 + *k.stride(),
100 + 0,
101 + *v.stride(),
102 + 0,
103 + *out.stride(),
104 + layout_crow_indices,
105 + layout_col_indices,
106 + *layout_crow_indices.stride(),
107 + *layout_col_indices.stride(),
108 + q_k_ratio,
109 + HAS_BATCH_DIM=False,
110 + D_HEAD=head_size,
111 + BLOCK_M=q_block_size,
112 + BLOCK_N=block_size,
113 + BLOCK_D=block_d,
114 + BLOCK_M_LOADING=(16 if decoding_only else
115 + q_block_size), # smaller for decoding
116 + EVEN_D=block_d == head_size,
117 + num_warps=1 if decoding_only else 4,
118 + num_stages=3)
119 +
120 + return out
121 +
122 +
123 +@triton.jit
124 +def _fwd_kernel_inner(
125 + acc,
126 + l_i,
127 + m_i,
128 + q,
129 + Q,
130 + k_block_col_idx,
131 + layout_col_ptr,
132 + layout_col_stride_h,
133 + layout_col_stride_m,
134 + k_ptrs,
135 + v_ptrs,
136 + off_h,
137 + offs_m,
138 + offs_n,
139 + offs_d,
140 + stride_kt,
141 + stride_vt,
142 + sm_scale,
143 + k_seqlen,
144 + past_len,
145 + LAST_K_BLOCK: tl.constexpr,
146 + BLOCK_M_LOADING: tl.constexpr,
147 + BLOCK_N: tl.constexpr,
148 + D_HEAD: tl.constexpr,
149 + EVEN_D: tl.constexpr,
150 + M_LT_N: tl.constexpr,
151 +):
152 + k_block_id = tl.load(layout_col_ptr + off_h * layout_col_stride_h +
153 + k_block_col_idx * layout_col_stride_m).to(tl.int32)
154 + start_n = k_block_id * BLOCK_N
155 + if LAST_K_BLOCK:
156 + if EVEN_D:
157 + k = tl.load(
158 + k_ptrs + start_n * stride_kt,
159 + mask=offs_n[None, :] + start_n < k_seqlen,
160 + )
161 + else:
162 + k = tl.load(
163 + k_ptrs + start_n * stride_kt,
164 + mask=(offs_n[None, :] + start_n < k_seqlen) &
165 + (offs_d[:, None] < D_HEAD),
166 + )
167 + else:
168 + if EVEN_D:
169 + k = tl.load(k_ptrs + start_n * stride_kt)
170 + else:
171 + k = tl.load(k_ptrs + start_n * stride_kt,
172 + mask=offs_d[:, None] < D_HEAD)
173 +
174 + qk = tl.zeros([BLOCK_M_LOADING, BLOCK_N], dtype=tl.float32)
175 + qk += tl.dot(q, k)
176 + qk *= sm_scale
177 +
178 + # the following is needed only when LAST_K_BLOCK or BLOCK_M < BLOCK_N
179 + if LAST_K_BLOCK | M_LT_N:
180 + qk += tl.where(
181 + offs_m[:, None] + past_len >= (start_n + offs_n[None, :]),
182 + 0,
183 + float("-inf"),
184 + )
185 +
186 + # flash-attn2
187 + m_ij = tl.maximum(m_i, tl.max(qk, 1))
188 + p = tl.math.exp2(qk - m_ij[:, None])
189 + l_ij = tl.sum(p, 1)
190 + alpha = tl.math.exp2(m_i - m_ij)
191 + acc = acc * alpha[:, None]
192 + # update m_i
193 + m_i = m_ij
194 + l_i = l_i * alpha + l_ij
195 +
196 + p = p.to(Q.dtype.element_ty)
197 + # update acc
198 + if LAST_K_BLOCK:
199 + if EVEN_D:
200 + v = tl.load(
201 + v_ptrs + start_n * stride_vt,
202 + mask=offs_n[:, None] + start_n < k_seqlen,
203 + )
204 + else:
205 + v = tl.load(
206 + v_ptrs + start_n * stride_vt,
207 + mask=(offs_n[:, None] + start_n < k_seqlen) &
208 + (offs_d[None, :] < D_HEAD),
209 + )
210 + else:
211 + if EVEN_D:
212 + v = tl.load(v_ptrs + start_n * stride_vt)
213 + else:
214 + v = tl.load(v_ptrs + start_n * stride_vt,
215 + mask=offs_d[None, :] < D_HEAD)
216 +
217 + acc += tl.dot(p, v)
218 +
219 + return acc, l_i, m_i
220 +
221 +
222 +@triton.heuristics({
223 + "M_LT_N":
224 + lambda kwargs: kwargs["BLOCK_M"] < kwargs["BLOCK_N"],
225 +})
226 +@triton.jit
227 +def _fwd_kernel_batch_inference(
228 + Q,
229 + K,
230 + V,
231 + Out,
232 + sm_scale,
233 + q_batch_starts,
234 + q_batch_ends,
235 + k_batch_starts,
236 + k_batch_ends,
237 + q_batch_ids,
238 + q_start_sids,
239 + stride_qb,
240 + stride_qt,
241 + stride_qh,
242 + stride_qd,
243 + stride_kb,
244 + stride_kt,
245 + stride_kh,
246 + stride_kd,
247 + stride_vb,
248 + stride_vt,
249 + stride_vh,
250 + stride_vd,
251 + stride_ob,
252 + stride_ot,
253 + stride_oh,
254 + stride_od,
255 + layout_crow_ptr,
256 + layout_col_ptr,
257 + layout_crow_stride_h,
258 + layout_crow_stride_m,
259 + layout_col_stride_h,
260 + layout_col_stride_m,
261 + q_k_ratio,
262 + HAS_BATCH_DIM: tl.constexpr,
263 + D_HEAD: tl.constexpr,
264 + BLOCK_M: tl.constexpr,
265 + BLOCK_N: tl.constexpr,
266 + BLOCK_D: tl.constexpr,
267 + BLOCK_M_LOADING: tl.constexpr,
268 + EVEN_D: tl.constexpr,
269 + M_LT_N: tl.constexpr,
270 +):
271 + """
272 + NOTATION:
273 + pid: position id
274 + sid: storage id
275 + sbid: storage block id
276 + pbid: position block id
277 + offs_m, offs_n: storage offsets of m-dim(q, row) and n-dim(k, col)
278 +
279 + TODO(linxihui):
280 + Optimize grouped-attn
281 + """
282 + off_zm = tl.program_id(0)
283 + off_h = tl.program_id(1)
284 +
285 + off_h_for_kv = off_h // q_k_ratio
286 +
287 + if HAS_BATCH_DIM:
288 + off_z = tl.program_id(2)
289 + Q += off_z * stride_qb
290 + K += off_z * stride_kb
291 + V += off_z * stride_vb
292 + Out += off_z * stride_ob
293 + start_m = off_zm
294 + q_start_sid = start_m * BLOCK_M # always 0 for decoding
295 + else:
296 + off_z = tl.load(q_batch_ids + off_zm).to(tl.int32) # [0, 0, 0, 1]
297 + q_start_sid = tl.load(q_start_sids + off_zm)
298 + start_m = q_start_sid // BLOCK_M # q_sbid
299 +
300 + offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M_LOADING)
301 + offs_n = tl.arange(0, BLOCK_N)
302 + offs_d = tl.arange(0, BLOCK_D)
303 +
304 + q_cu_start = tl.load(q_batch_starts + off_z).to(tl.int32)
305 + q_seqlen = tl.load(q_batch_ends + off_z).to(tl.int32) - q_cu_start
306 + k_cu_start = tl.load(k_batch_starts + off_z).to(tl.int32)
307 + k_seqlen = tl.load(k_batch_ends + off_z).to(tl.int32) - k_cu_start
308 + past_len = k_seqlen - q_seqlen
309 +
310 + Q += q_cu_start * stride_qt + off_h * stride_qh
311 + K += k_cu_start * stride_kt + off_h_for_kv * stride_kh
312 + V += k_cu_start * stride_vt + off_h_for_kv * stride_vh
313 + Out += q_cu_start * stride_ot + off_h * stride_oh
314 +
315 + q_pbid = (past_len + q_start_sid) // BLOCK_M
316 +
317 + if EVEN_D:
318 + q = tl.load(
319 + Q + offs_m[:, None] * stride_qt + offs_d[None, :] * stride_qd,
320 + mask=offs_m[:, None] < q_seqlen,
321 + )
322 + else:
323 + q = tl.load(
324 + Q + offs_m[:, None] * stride_qt + offs_d[None, :] * stride_qd,
325 + mask=(offs_m[:, None] < q_seqlen) & (offs_d[None, :] < D_HEAD),
326 + other=0,
327 + )
328 +
329 + sparse_crow_ptr = (layout_crow_ptr + off_h * layout_crow_stride_h +
330 + q_pbid * layout_crow_stride_m)
331 +
332 + # TODO(linxihui): load at once, with any Triton version
333 + # that supports `tl.split`, e.g., Triton 3.0
334 + k_block_start = tl.load(sparse_crow_ptr).to(tl.int32)
335 + k_block_end = tl.load(sparse_crow_ptr + 1).to(tl.int32)
336 +
337 + m_i = tl.zeros([BLOCK_M_LOADING], dtype=tl.float32) - float("inf")
338 + l_i = tl.zeros([BLOCK_M_LOADING], dtype=tl.float32)
339 + acc = tl.zeros([BLOCK_M_LOADING, BLOCK_D], dtype=tl.float32)
340 +
341 + k_ptrs = K + offs_n[None, :] * stride_kt + offs_d[:, None] * stride_kd
342 + v_ptrs = V + offs_n[:, None] * stride_vt + offs_d[None, :] * stride_vd
343 +
344 + sm_scale *= (
345 + 1.44269504 # 1/log2 as we use base2 for exponential and logarithm
346 + )
347 +
348 + for k_block_col_idx in range(k_block_start, k_block_end - 1):
349 + acc, l_i, m_i = _fwd_kernel_inner(
350 + acc,
351 + l_i,
352 + m_i,
353 + q,
354 + Q,
355 + k_block_col_idx,
356 + layout_col_ptr,
357 + layout_col_stride_h,
358 + layout_col_stride_m,
359 + k_ptrs,
360 + v_ptrs,
361 + off_h,
362 + offs_m,
363 + offs_n,
364 + offs_d,
365 + stride_kt,
366 + stride_vt,
367 + sm_scale,
368 + k_seqlen,
369 + past_len,
370 + False,
371 + BLOCK_M_LOADING,
372 + BLOCK_N,
373 + D_HEAD,
374 + EVEN_D,
375 + M_LT_N,
376 + )
377 +
378 + acc, l_i, m_i = _fwd_kernel_inner(
379 + acc,
380 + l_i,
381 + m_i,
382 + q,
383 + Q,
384 + k_block_end - 1,
385 + layout_col_ptr,
386 + layout_col_stride_h,
387 + layout_col_stride_m,
388 + k_ptrs,
389 + v_ptrs,
390 + off_h,
391 + offs_m,
392 + offs_n,
393 + offs_d,
394 + stride_kt,
395 + stride_vt,
396 + sm_scale,
397 + k_seqlen,
398 + past_len,
399 + True,
400 + BLOCK_M_LOADING,
401 + BLOCK_N,
402 + D_HEAD,
403 + EVEN_D,
404 + M_LT_N,
405 + )
406 +
407 + # flash-attn 2
408 + m_i += tl.math.log2(l_i)
409 + acc = acc / l_i[:, None]
410 +
411 + # write output
412 + if EVEN_D:
413 + tl.store(
414 + Out + offs_m[:, None] * stride_ot + offs_d[None, :] * stride_od,
415 + acc,
416 + mask=offs_m[:, None] < q_seqlen,
417 + )
418 + else:
419 + tl.store(
420 + Out + offs_m[:, None] * stride_ot + offs_d[None, :] * stride_od,
421 + acc,
422 + mask=(offs_m[:, None] < q_seqlen) & (offs_d[None, :] < D_HEAD),
423 + )
vllm/attention/ops/blocksparse_attention/interface.py new
+238
@@ -0,0 +1,238 @@
1 +import math
2 +
3 +import torch
4 +
5 +from vllm.utils import is_cpu, is_hip
6 +
7 +from .utils import (dense_to_crow_col, get_head_sliding_step,
8 + get_sparse_attn_mask)
9 +
10 +IS_COMPUTE_8_OR_ABOVE = (torch.cuda.is_available()
11 + and torch.cuda.get_device_capability()[0] >= 8)
12 +
13 +if IS_COMPUTE_8_OR_ABOVE:
14 + from .blocksparse_attention_kernel import blocksparse_flash_attn_varlen_fwd
15 +
16 +
17 +class LocalStridedBlockSparseAttn(torch.nn.Module):
18 +
19 + def __init__(
20 + self,
21 + n_heads,
22 + max_seqlen,
23 + local_blocks,
24 + vert_stride,
25 + block_size,
26 + device=None,
27 + dtype=None,
28 + homo_head=False,
29 + active_head_range=None,
30 + q_block_size=None,
31 + use_spda=None,
32 + ):
33 + super().__init__()
34 + if use_spda is None:
35 + use_spda = is_hip() or is_cpu() or not \
36 + IS_COMPUTE_8_OR_ABOVE
37 + device = device or (torch.cuda.current_device()
38 + if torch.cuda.is_available() else "cpu")
39 + device = torch.device(device)
40 + # NOTE: vllm CPU backend support BF16 instead of FP16.
41 + dtype = dtype or (torch.bfloat16 if IS_COMPUTE_8_OR_ABOVE
42 + or device.type == "cpu" else torch.half)
43 +
44 + self.n_heads = n_heads
45 + self.max_seqlen = max_seqlen
46 + self.local_blocks = local_blocks
47 + self.vert_stride = vert_stride
48 + self.use_spda = use_spda
49 + self.dtype = dtype
50 + self.device = device
51 + self.block_size = block_size
52 + self.q_block_size = q_block_size
53 + self.homo_head = homo_head
54 + self.active_head_range = active_head_range
55 + self.head_sliding_step = get_head_sliding_step(n_heads, vert_stride,
56 + homo_head)
57 +
58 + sparse_layout, sparse_pattern, self.dense_attn_mask = (
59 + self.get_attn_pattern(dtype, device))
60 +
61 + if q_block_size is not None and q_block_size != block_size:
62 + if q_block_size > block_size:
63 + assert q_block_size % block_size == 0
64 + blocks_to_merge = q_block_size // block_size
65 + shape = sparse_pattern.shape
66 + sparse_pattern = sparse_pattern.view(shape[0], -1,
67 + blocks_to_merge,
68 + shape[-1])
69 + sparse_pattern = sparse_pattern.sum(2)
70 + sparse_layout = dense_to_crow_col(sparse_pattern)
71 + else:
72 + raise ValueError(
73 + "Does not support smaller q_block_size. It will be slower."
74 + )
75 +
76 + self.sparse_layout = sparse_layout
77 +
78 + def get_attn_pattern(self, dtype, device):
79 + sparse_layout, sparse_pattern, dense_attn_mask = get_sparse_attn_mask(
80 + self.n_heads,
81 + self.max_seqlen,
82 + self.max_seqlen,
83 + dtype,
84 + device,
85 + block_size=self.block_size,
86 + local_blocks=self.local_blocks,
87 + vert_stride=self.vert_stride,
88 + homo_head=self.homo_head,
89 + return_dense=self.use_spda,
90 + dense_mask_type="bias",
91 + )
92 + if (not self.homo_head) and (self.active_head_range is not None):
93 + assert isinstance(self.active_head_range, tuple)
94 + assert (len(self.active_head_range) == 2)
95 + h_start, h_end = self.active_head_range
96 + sparse_layout = tuple(x[h_start:h_end] for x in sparse_layout)
97 + if self.use_spda:
98 + dense_attn_mask = dense_attn_mask[h_start:h_end]
99 + return sparse_layout, sparse_pattern, dense_attn_mask
100 +
101 + def varlen_attn(self,
102 + q,
103 + k,
104 + v,
105 + cu_seqlens_k,
106 + cu_seqlens_q=None,
107 + sm_scale=None):
108 + """
109 + q, k, v: shape = (num_tokens, num_heads_q/kv, head_size).
110 + Support grouped attention, with `q[:, i*r:(i*r + r)]`
111 + is correspondent to `k[:, i]`, where `r` is the q/k ratio.
112 + cu_seqlens_k: shape=(batch_size + 1,),
113 + indicating segment of samples,
114 + e.g., `k[cu_seqlen[i]:cu_seqlne[i+1]]` is q of sample i
115 + cu_seqlens_q: shape=(batch_size + 1, ).
116 + Default None: same as cu_seqlens_k for prefilling or
117 + [0, 1, .., batch_size] for decoding.
118 + The only case you need to specify is when q is a mix of
119 + prefilling and decoding.
120 + sm_scale: softmax scale, default to 1/sqrt(head_size).
121 +
122 + return: tensor of shape as q.
123 + """
124 + assert (
125 + IS_COMPUTE_8_OR_ABOVE
126 + ), "Requires compute capability of 8 or above (Ampere or newer) to use \
127 + Triton kernel."
128 +
129 + sm_scale = sm_scale or 1.0 / math.sqrt(q.size(-1))
130 +
131 + return blocksparse_flash_attn_varlen_fwd(
132 + q,
133 + k,
134 + v,
135 + cu_seqlens_k,
136 + cu_seqlens_q,
137 + sm_scale,
138 + self.sparse_layout,
139 + block_size=self.block_size,
140 + q_block_size=self.q_block_size,
141 + max_seqlen=self.max_seqlen,
142 + )
143 +
144 + @staticmethod
145 + def transpose_and_pad(x, cu_seqlens, maxlen, head_repeats=1):
146 + """
147 + :param x: (total_tokens, n_heads, head_size)
148 + :return: (batch, n_heads, length, head_size)
149 + """
150 + x_padded = x.new_empty(
151 + len(cu_seqlens) - 1, x.size(1), head_repeats, maxlen, x.size(2))
152 + cu_seqlens = cu_seqlens.cpu()
153 + for i, (s, e) in enumerate(zip(cu_seqlens[:-1], cu_seqlens[1:])):
154 + x_padded[i, :, :, :e - s].copy_(x[s:e].transpose(0,
155 + 1).unsqueeze(1))
156 + return x_padded.flatten(1, 2)
157 +
158 + @staticmethod
159 + def transpose_and_unpad(x_padded, cu_seqlens):
160 + """
161 + :param x_padded: (batch, n_heads, length, head_size)
162 + :return: (total_tokens, n_heads, head_size)
163 + """
164 + cu_seqlens = cu_seqlens.cpu()
165 + total_n_tokens = cu_seqlens[-1]
166 + x = x_padded.new_empty(total_n_tokens, x_padded.size(1),
167 + x_padded.size(3))
168 + for i, (s, e) in enumerate(zip(cu_seqlens[:-1], cu_seqlens[1:])):
169 + x[s:e].copy_(x_padded[i, :, :e - s].transpose(0, 1))
170 + return x
171 +
172 + def spda(self, q, k, v, cu_seqlens_k, cu_seqlens_q=None, sm_scale=None):
173 + """For CPU, V100 or other older GPUs.
174 + NOTE: torch SPDA supports nested tensor,
175 + but seems extremely slow. Choose to pad instead.
176 + """
177 + assert (cu_seqlens_q is None or
178 + (cu_seqlens_q
179 + == cu_seqlens_k).all()), "Can only handle prompt with SPDA."
180 + assert q.size(0) == k.size(0), "can only handle prompt with SPDA."
181 +
182 + assert q.size(1) % k.size(1) == 0
183 + q_k_ratio = q.size(1) // k.size(1)
184 + sm_scale = sm_scale or 1.0 / math.sqrt(q.size(-1))
185 + cu_seqlens = cu_seqlens_k.cpu()
186 + maxlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
187 +
188 + if (self.dense_attn_mask.dtype != q.dtype
189 + or self.dense_attn_mask.device != q.device):
190 + _, _, self.dense_attn_mask = self.get_attn_pattern(
191 + q.dtype, q.device)
192 + attn_mask = self.dense_attn_mask[None, :, :maxlen, :maxlen]
193 +
194 + q2 = self.transpose_and_pad(q, cu_seqlens, maxlen, 1)
195 + k2, v2 = [
196 + self.transpose_and_pad(x, cu_seqlens, maxlen, q_k_ratio)
197 + for x in [k, v]
198 + ]
199 + spda_output = torch.nn.functional.scaled_dot_product_attention(
200 + q2, k2, v2, attn_mask=attn_mask, scale=sm_scale)
201 + return self.transpose_and_unpad(spda_output, cu_seqlens)
202 +
203 + def forward(self, q, k, v, cu_seqlens_k, cu_seqlens_q=None, sm_scale=None):
204 + """Dispatch to `varlen_attn` (Ampere or newer) or
205 + `self.spda`(cpu, Volta, Turing or older)based on
206 + the type of device used and cuda compute capability.
207 +
208 + q, k, v: shape = (num_tokens, num_heads_q/kv, head_size).
209 + Support grouped attention, with `q[:, i*r:(i*r + r)]`
210 + is correspondent to `k[:, i]`, where `r` is the q/k ratio.
211 + cu_seqlens_k: shape=(batch_size + 1,), indicating segment of samples,
212 + e.g., `k[cu_seqlen[i]:cu_seqlne[i+1]]` is q of sample i
213 + cu_seqlens_q: shape=(batch_size + 1, ).
214 + Default None: same as cu_seqlens_k for prefilling or
215 + [0, 1, .., batch_size] for decoding.
216 + The only case you need to specify
217 + is when q is a mix of prefilling
218 + and decoding.
219 + sm_scale: softmax scale, default to 1/sqrt(head_size).
220 +
221 + return: tensor of shape as q.
222 + """
223 + assert k.dim() == 3
224 + if self.use_spda:
225 + return self.spda(
226 + q,
227 + k,
228 + v,
229 + cu_seqlens_k,
230 + cu_seqlens_q=cu_seqlens_q,
231 + sm_scale=sm_scale,
232 + )
233 + return self.varlen_attn(q,
234 + k,
235 + v,
236 + cu_seqlens_k,
237 + cu_seqlens_q=cu_seqlens_q,
238 + sm_scale=sm_scale)
\ No newline at end of file
vllm/attention/ops/blocksparse_attention/utils.py new
+216
@@ -0,0 +1,216 @@
1 +# Helper functions for 3D sparse pattern
2 +# These function are not optimized and very inefficient.
3 +# Avoid calling them too frequent or use a cache mechanism.
4 +
5 +from functools import lru_cache
6 +
7 +import torch
8 +import triton
9 +from scipy import sparse
10 +
11 +
12 +def dense_to_crow_col(x: torch.Tensor):
13 + """Turning a 2D/3D torch tensor (x) to CSR rows/cols indexing.
14 + NOTE: col_indices padded -1
15 + """
16 + device = x.device
17 + pad = -1
18 + dim = x.dim()
19 + assert x.dim() in (2, 3)
20 + if x.dim() == 2:
21 + x = x[None]
22 + x = [sparse.csr_matrix(xi.bool().cpu().numpy()) for xi in x]
23 + crows = torch.vstack([torch.from_numpy(xi.indptr) for xi in x])
24 + cols = [torch.from_numpy(xi.indices) for xi in x]
25 + max_cols = max(len(xi) for xi in cols)
26 + cols = [
27 + torch.cat([xi, pad + xi.new_zeros(max_cols - xi.shape[0])])
28 + for xi in cols
29 + ]
30 + cols = torch.vstack(cols)
31 + if dim == 2:
32 + crows = crows[0]
33 + cols = cols[0]
34 + return crows.to(device), cols.to(device)
35 +
36 +
37 +def crow_col_to_dense(crows: torch.Tensor,
38 + cols: torch.Tensor,
39 + dtype: torch.dtype = torch.float16):
40 + dim = crows.dim()
41 + if dim == 1:
42 + crows = crows[None]
43 + cols = cols[None]
44 + device = crows.device
45 + crows, cols = crows.cpu(), cols.cpu() # faster in cpu
46 + shape = (crows.shape[0], crows.shape[1] - 1, cols.max() + 1)
47 + x = torch.zeros(shape, dtype=dtype)
48 + for i in range(shape[0]):
49 + for j in range(shape[1]):
50 + x[i, j, cols[i, crows[i, j]:crows[i, j + 1]]] = 1
51 + if dim == 1:
52 + x = x[0]
53 + return x.to(device)
54 +
55 +
56 +def dense_to_ccol_row(x: torch.Tensor):
57 + """Similar, but to CSC format"""
58 + x = x.transpose(-2, -1)
59 + return dense_to_crow_col(x)
60 +
61 +
62 +def ccol_row_to_dense(ccol: torch.Tensor,
63 + rows: torch.Tensor,
64 + dtype: torch.dtype = torch.float16):
65 + return crow_col_to_dense(ccol, rows, dtype).permute(0, 2, 1).contiguous()
66 +
67 +
68 +def _get_sparse_attn_mask_homo_head(
69 + q_len: int,
70 + max_seqlen: int,
71 + dtype: torch.dtype,
72 + device: torch.device,
73 + block_size: int = 128,
74 + local_blocks: int = 4,
75 + vert_stride: int = 4,
76 + return_dense: bool = False,
77 +):
78 + """
79 + :return: a tuple of 3:
80 + - tuple of crow_indices, col_indices representation
81 + of CSR format.
82 + - block dense mask
83 + - all token dense mask (be aware that it can be
84 + OOM if it is too big) if `return_dense==True`,
85 + otherwise, None
86 + """
87 + with torch.no_grad():
88 + num_blocks = triton.cdiv(max_seqlen, block_size)
89 + q_pos = torch.arange(num_blocks)[:, None]
90 + k_pos = torch.arange(num_blocks)[None]
91 + mask_vert_strided = (torch.arange(num_blocks) + 1) % vert_stride == 0
92 + block_mask_dense = (((q_pos >= k_pos)
93 + & ((q_pos - k_pos < local_blocks)
94 + | mask_vert_strided)).to(device).to(dtype))
95 + num_blocks_q = triton.cdiv(q_len, block_size)
96 + block_mask_dense_output = (dense_to_crow_col(
97 + block_mask_dense[-num_blocks_q:].contiguous()))
98 + if return_dense:
99 + mask_dense = torch.kron(
100 + block_mask_dense,
101 + block_mask_dense.new_ones((block_size, block_size)),
102 + )
103 + causal_mask = torch.tril(torch.ones(
104 + max_seqlen, max_seqlen)).type_as(mask_dense)[-q_len:]
105 + mask_dense = mask_dense[-q_len:, :max_seqlen] * causal_mask
106 + return (
107 + block_mask_dense_output,
108 + block_mask_dense,
109 + mask_dense,
110 + )
111 + else:
112 + return (
113 + block_mask_dense_output,
114 + block_mask_dense,
115 + None,
116 + )
117 +
118 +
119 +def binary_mask_to_bias(mask_dense: torch.Tensor):
120 + mask_dense = 1 - mask_dense
121 + mask_dense.masked_fill_(mask_dense.bool(), -torch.inf)
122 + return mask_dense
123 +
124 +
125 +def get_head_sliding_step(n_heads: int,
126 + vert_stride: int,
127 + homo_head: bool = False):
128 + if homo_head:
129 + return 0
130 + return max(1, int(vert_stride / n_heads))
131 +
132 +
133 +@lru_cache
134 +def get_sparse_attn_mask(
135 + n_heads: int,
136 + q_len: int,
137 + max_seqlen: int,
138 + dtype: torch.dtype,
139 + device: torch.device,
140 + block_size: int = 64,
141 + local_blocks: int = 4,
142 + vert_stride: int = 4,
143 + homo_head: bool = True,
144 + return_dense: bool = False,
145 + dense_mask_type: str = "binary",
146 +):
147 + """
148 + :param dense_mask_type: "binary" (0 for skip token, 1 for others)
149 + or "bias" (-inf for skip token, 0 or others)
150 + :return: a tuple of 3:
151 + - tuple of crow_indices, col_indices representation
152 + of CSR format.
153 + - block dense mask
154 + - all token dense mask (be aware that it can be OOM if it
155 + is too big) if `return_dense==True`, otherwise, None
156 + """
157 + assert dense_mask_type in ("binary", "bias")
158 + if homo_head:
159 + with torch.no_grad():
160 + (crow, col), block_mask_dense, mask_dense = (
161 + _get_sparse_attn_mask_homo_head(
162 + q_len,
163 + max_seqlen,
164 + dtype,
165 + device,
166 + block_size,
167 + local_blocks,
168 + vert_stride,
169 + return_dense,
170 + ))
171 + crow = crow[None].expand(n_heads, crow.shape[0])
172 + col = col[None].expand(n_heads, col.shape[0])
173 + if return_dense:
174 + mask_dense = mask_dense[None].expand(n_heads,
175 + *mask_dense.shape)
176 + if dense_mask_type == "bias":
177 + mask_dense = binary_mask_to_bias(mask_dense)
178 + return (crow, col), block_mask_dense, mask_dense
179 +
180 + with torch.no_grad():
181 + num_blocks = triton.cdiv(max_seqlen, block_size)
182 + q_pos = torch.arange(num_blocks)[None, :, None]
183 + k_pos = torch.arange(num_blocks)[None, None]
184 + head_sliding_step = get_head_sliding_step(n_heads, vert_stride)
185 + mask_vert_strided = [
186 + (torch.arange(num_blocks) + h * head_sliding_step + 1) %
187 + vert_stride == 0 for h in range(n_heads)
188 + ]
189 + mask_vert_strided = torch.vstack(mask_vert_strided).unsqueeze(1)
190 + block_mask_dense = (((q_pos >= k_pos)
191 + & ((q_pos - k_pos < local_blocks)
192 + | mask_vert_strided)).to(device).to(dtype))
193 + num_blocks_q = triton.cdiv(q_len, block_size)
194 + block_mask_dense_output = block_mask_dense[:, -num_blocks_q:]
195 + if return_dense:
196 + mask_dense = torch.kron(
197 + block_mask_dense,
198 + block_mask_dense.new_ones((block_size, block_size)),
199 + )
200 + causal_mask = torch.tril(torch.ones(
201 + max_seqlen, max_seqlen)).type_as(mask_dense)[-q_len:]
202 + mask_dense = mask_dense[..., -q_len:, :max_seqlen] * causal_mask[None]
203 + if dense_mask_type == "bias":
204 + mask_dense = binary_mask_to_bias(mask_dense)
205 +
206 + return (
207 + dense_to_crow_col(block_mask_dense_output),
208 + block_mask_dense,
209 + mask_dense,
210 + )
211 + else:
212 + return (
213 + dense_to_crow_col(block_mask_dense_output),
214 + block_mask_dense,
215 + None,
216 + )
vllm/attention/ops/paged_attn.py
+24 -1
@@ -91,9 +91,21 @@ class PagedAttention:
91 scale: float,
92 alibi_slopes: Optional[torch.Tensor],
93 kv_scale: float,
94 + tp_rank: int = 0,
95 + blocksparse_local_blocks: int = 0,
96 + blocksparse_vert_stride: int = 0,
97 + blocksparse_block_size: int = 64,
98 + blocksparse_head_sliding_step: int = 0,
99 ) -> torch.Tensor:
95 - output = torch.empty_like(query)
100 + if blocksparse_vert_stride is not None and blocksparse_vert_stride > 1:
101 + # use blocksparse paged attention
102 + block_size = value_cache.size(-1)
103 + assert (blocksparse_block_size > 0 and
104 + blocksparse_block_size % block_size == 0), \
105 + (f"{blocksparse_block_size=} needs to be a multiple of"
106 + f"{block_size=} used in block_tables.")
107
108 + output = torch.empty_like(query)
109 block_size = value_cache.shape[3]
110 num_seqs, num_heads, head_size = query.shape
111 max_num_partitions = ((max_seq_len + _PARTITION_SIZE - 1) //
@@ -107,6 +119,7 @@ class PagedAttention:
119 # For context len > 8192, use V2 kernel to avoid shared memory shortage.
120 use_v1 = (max_seq_len <= 8192
121 and (max_num_partitions == 1 or num_seqs * num_heads > 512))
122 +
123 if use_v1:
124 # Run PagedAttention V1.
125 ops.paged_attention_v1(
@@ -123,6 +136,11 @@ class PagedAttention:
136 alibi_slopes,
137 kv_cache_dtype,
138 kv_scale,
139 + tp_rank,
140 + blocksparse_local_blocks,
141 + blocksparse_vert_stride,
142 + blocksparse_block_size,
143 + blocksparse_head_sliding_step,
144 )
145 else:
146 # Run PagedAttention V2.
@@ -155,6 +173,11 @@ class PagedAttention:
173 alibi_slopes,
174 kv_cache_dtype,
175 kv_scale,
176 + tp_rank,
177 + blocksparse_local_blocks,
178 + blocksparse_vert_stride,
179 + blocksparse_block_size,
180 + blocksparse_head_sliding_step,
181 )
182 return output
183
vllm/attention/selector.py
+7
@@ -29,7 +29,14 @@ def get_attn_backend(
29 dtype: torch.dtype,
30 kv_cache_dtype: Optional[str],
31 block_size: int,
32 + is_blocksparse: bool = False,
33 ) -> Type[AttentionBackend]:
34 +
35 + if is_blocksparse:
36 + logger.info("Using BlocksparseFlashAttention backend.")
37 + from vllm.attention.backends.blocksparse_attn import (
38 + BlocksparseFlashAttentionBackend)
39 + return BlocksparseFlashAttentionBackend
40 """Determine which attention backend to use and only import
41 the selected backend module.
42 """
vllm/entrypoints/openai/serving_engine.py
+1
@@ -100,6 +100,7 @@ class OpenAIServing:
100 token_logprob = step_top_logprobs[token_id].logprob
101 token = step_top_logprobs[token_id].decoded_token
102 logprobs.tokens.append(token)
103 + token_logprob = max(token_logprob, -9999.0)
104 logprobs.token_logprobs.append(token_logprob)
105
106 if num_output_top_logprobs:
vllm/model_executor/models/__init__.py
+1
@@ -56,6 +56,7 @@ _GENERATION_MODELS = {
56 "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
57 "ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
58 "XverseForCausalLM": ("xverse", "XverseForCausalLM"),
59 + "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
60 }
61
62 _EMBEDDING_MODELS = {
vllm/model_executor/models/phi3_small.py new
+447
@@ -0,0 +1,447 @@
1 +import math
2 +from typing import Iterable, List, Optional, Tuple
3 +
4 +import torch
5 +from torch import nn
6 +from transformers.configuration_utils import PretrainedConfig
7 +
8 +from vllm.attention import Attention, AttentionMetadata
9 +from vllm.config import CacheConfig, LoRAConfig
10 +from vllm.distributed import (get_tensor_model_parallel_rank,
11 + get_tensor_model_parallel_world_size)
12 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
13 + QKVParallelLinear,
14 + RowParallelLinear)
15 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
16 +from vllm.model_executor.layers.quantization.base_config import (
17 + QuantizationConfig)
18 +from vllm.model_executor.layers.rotary_embedding import get_rope
19 +from vllm.model_executor.layers.sampler import Sampler
20 +from vllm.model_executor.layers.vocab_parallel_embedding import (
21 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
22 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23 +from vllm.model_executor.sampling_metadata import SamplingMetadata
24 +from vllm.sequence import SamplerOutput
25 +
26 +
27 +def load_column_parallel_weight(param: torch.nn.Parameter,
28 + loaded_weight: torch.Tensor):
29 + tp = get_tensor_model_parallel_world_size()
30 + rk = get_tensor_model_parallel_rank()
31 + assert param.size(0) * tp == loaded_weight.size(0)
32 + s = rk * param.size(0)
33 + e = (rk + 1) * param.size(0)
34 + loaded_weight = loaded_weight[s:e]
35 + assert param.shape == loaded_weight.shape
36 + param.data.copy_(loaded_weight)
37 +
38 +
39 +class HeadMajorQKVParallelLinear(QKVParallelLinear):
40 +
41 + def weight_loader(self, param: torch.nn.Parameter,
42 + loaded_weight: torch.Tensor):
43 + return load_column_parallel_weight(param, loaded_weight)
44 +
45 +
46 +class HeadMajorColumnParallelLinear(MergedColumnParallelLinear):
47 +
48 + def weight_loader(self, param: torch.nn.Parameter,
49 + loaded_weight: torch.Tensor):
50 + return load_column_parallel_weight(param, loaded_weight)
51 +
52 +
53 +@torch.jit.script
54 +def quick_gelu(x):
55 + return x * torch.sigmoid(1.702 * x)
56 +
57 +
58 +@torch.jit.script
59 +def gegelu(input, limit: Optional[float] = None):
60 + a_gelu, a_linear = input[..., ::2], input[..., 1::2]
61 + if limit is not None:
62 + a_gelu = torch.where(torch.isinf(a_gelu), a_gelu,
63 + a_gelu.clamp(min=None, max=limit))
64 + a_linear = torch.where(
65 + torch.isinf(a_linear),
66 + a_linear,
67 + a_linear.clamp(min=-limit, max=limit),
68 + )
69 + out_gelu = quick_gelu(a_gelu)
70 + return out_gelu * (a_linear + 1)
71 +
72 +
73 +class Phi3SmallMLP(nn.Module):
74 +
75 + def __init__(
76 + self,
77 + config: PretrainedConfig,
78 + quant_config: Optional[QuantizationConfig] = None,
79 + ) -> None:
80 + super().__init__()
81 + self.config = config
82 + assert (self.config.hidden_act == "gegelu"
83 + ), "Only `gegelu` is supported for the 4.7 series of models .."
84 + self.hidden_size = config.hidden_size
85 + self.gegelu_limit = config.gegelu_limit
86 + self.intermediate_size = config.intermediate_size
87 +
88 + self.up_proj = HeadMajorColumnParallelLinear(
89 + self.hidden_size,
90 + 2 * [self.intermediate_size],
91 + bias=True,
92 + quant_config=quant_config,
93 + )
94 + self.down_proj = RowParallelLinear(
95 + self.intermediate_size,
96 + self.hidden_size,
97 + bias=True,
98 + quant_config=quant_config,
99 + )
100 +
101 + def forward(self, x):
102 + gate_up, _ = self.up_proj(x)
103 + x = gegelu(gate_up)
104 + x, _ = self.down_proj(x)
105 + return x
106 +
107 +
108 +class Phi3SmallSelfAttention(nn.Module):
109 +
110 + def __init__(
111 + self,
112 + config: PretrainedConfig,
113 + layer_idx: int,
114 + cache_config: Optional[CacheConfig] = None,
115 + quant_config: Optional[QuantizationConfig] = None,
116 + ) -> None:
117 + super().__init__()
118 + self.layer_idx = layer_idx
119 + self.config = config
120 + self.sparse_block_size = config.blocksparse_block_size
121 + self.homo_heads = config.blocksparse_homo_head_pattern
122 + self.local_blocks = config.blocksparse_num_local_blocks
123 + self.vert_stride = config.blocksparse_vert_stride
124 +
125 + assert (config.blocksparse_block_size ==
126 + config.blocksparse_triton_kernel_block_size)
127 +
128 + self.hidden_size = config.hidden_size
129 + # Number of Query Heads
130 + self.num_heads = config.num_attention_heads
131 +
132 + self.head_dim = self.hidden_size // self.num_heads
133 + self.tp_size = get_tensor_model_parallel_world_size()
134 + # Number of total Key Value Heads before tensor parallel
135 + self.num_key_value_heads = config.num_key_value_heads
136 + self.num_q_per_kv = self.num_heads // self.num_key_value_heads
137 + if self.tp_size > 1:
138 + assert self.num_key_value_heads % self.tp_size == 0
139 + self.num_kv_heads_per_partion = max(
140 + 1, self.num_key_value_heads // self.tp_size)
141 + self.num_heads_per_partition = self.num_heads // self.tp_size
142 +
143 + self.max_position_embeddings = config.max_position_embeddings
144 + self.rope_embedding_base = config.rope_embedding_base
145 + self.rope_position_scale = config.rope_position_scale
146 + self.is_causal = True
147 +
148 + norm_factor = None
149 + if config.mup_use_scaling:
150 + norm_factor = self.head_dim / config.mup_attn_multiplier
151 + else:
152 + norm_factor = math.sqrt(self.head_dim)
153 + self.scale = 1 / norm_factor
154 +
155 + self.query_key_value = HeadMajorQKVParallelLinear(
156 + self.hidden_size,
157 + self.head_dim,
158 + self.num_heads,
159 + self.num_key_value_heads,
160 + bias=True,
161 + quant_config=quant_config,
162 + )
163 +
164 + self.dense = RowParallelLinear(self.hidden_size,
165 + self.hidden_size,
166 + bias=True,
167 + quant_config=quant_config)
168 +
169 + if getattr(self.config, "rope_scaling", None) is not None:
170 + rope_scaling = self.config.rope_scaling
171 + for key in rope_scaling:
172 + if isinstance(rope_scaling[key], list):
173 + rope_scaling[key] = tuple(rope_scaling[key])
174 +
175 + if "factor" not in rope_scaling:
176 + rope_scaling["factor"] = self.rope_position_scale
177 + else:
178 + rope_scaling = {
179 + "type": "linear",
180 + "factor": self.rope_position_scale,
181 + }
182 +
183 + self.rotary_emb = get_rope(
184 + self.head_dim,
185 + rotary_dim=self.head_dim,
186 + max_position=self.max_position_embeddings,
187 + base=self.rope_embedding_base,
188 + rope_scaling=rope_scaling,
189 + )
190 +
191 + # blocksparse params
192 + self.blocksparse_block_size = config.blocksparse_block_size
193 + self.blocksparse_num_local_blocks = config.blocksparse_num_local_blocks
194 + self.blocksparse_vert_stride = config.blocksparse_vert_stride
195 +
196 + use_dense_attn = (getattr(self.config,
197 + "dense_attention_every_n_layers", None)
198 + and (self.layer_idx + 1) %
199 + self.config.dense_attention_every_n_layers == 0)
200 +
201 + bs_params = None
202 + if not use_dense_attn:
203 + bs_params = {
204 + 'max_seqlen': self.max_position_embeddings,
205 + 'num_heads': self.num_heads_per_partition,
206 + "num_kv_heads": self.num_kv_heads_per_partion,
207 + "block_size": self.sparse_block_size,
208 + "local_blocks": self.local_blocks,
209 + "vert_stride": self.vert_stride,
210 + "homo_head": self.homo_heads
211 + }
212 +
213 + self.attn = Attention(
214 + self.num_heads_per_partition,
215 + self.head_dim,
216 + self.scale,
217 + num_kv_heads=self.num_kv_heads_per_partion,
218 + cache_config=cache_config,
219 + quant_config=quant_config,
220 + blocksparse_params=bs_params,
221 + )
222 +
223 + def forward(
224 + self,
225 + positions: torch.Tensor,
226 + hidden_states: torch.Tensor,
227 + kv_cache: torch.Tensor,
228 + attn_metadata: AttentionMetadata,
229 + ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
230 + Optional[Tuple[torch.Tensor]]]:
231 + qkv, _ = self.query_key_value(hidden_states)
232 +
233 + qkv = qkv.view(qkv.shape[:-1] +
234 + (-1, (self.num_q_per_kv + 2), self.head_dim))
235 + q, k, v = qkv.split([self.num_q_per_kv, 1, 1], dim=-2)
236 +
237 + # NOTE: this is required by RotaryEmbed, which indeed does not have to
238 + # TODO: allow 3D QK for rotary forward
239 + q = q.reshape(-1, self.head_dim * self.num_heads_per_partition)
240 + k = k.reshape(-1, self.head_dim * self.num_kv_heads_per_partion)
241 + v = v.reshape(-1, self.head_dim * self.num_kv_heads_per_partion)
242 +
243 + q, k = self.rotary_emb(positions, q, k)
244 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata=attn_metadata)
245 + output, _ = self.dense(attn_output)
246 +
247 + return output
248 +
249 +
250 +class Phi3SmallDecoderLayer(nn.Module):
251 +
252 + def __init__(
253 + self,
254 + config: PretrainedConfig,
255 + layer_idx: int,
256 + cache_config: Optional[CacheConfig] = None,
257 + quant_config: Optional[QuantizationConfig] = None,
258 + ):
259 + super().__init__()
260 + self.hidden_size = config.hidden_size
261 + self.self_attn = Phi3SmallSelfAttention(config,
262 + layer_idx,
263 + cache_config=cache_config,
264 + quant_config=quant_config)
265 + self.mlp = Phi3SmallMLP(config, quant_config)
266 +
267 + self.input_layernorm = nn.LayerNorm(config.hidden_size,
268 + eps=config.layer_norm_epsilon)
269 + self.post_attention_layernorm = nn.LayerNorm(
270 + config.hidden_size, eps=config.layer_norm_epsilon)
271 +
272 + def forward(
273 + self,
274 + positions: torch.Tensor,
275 + hidden_states: torch.Tensor,
276 + kv_cache: torch.Tensor,
277 + attn_metadata: AttentionMetadata,
278 + ) -> torch.Tensor:
279 + residual = hidden_states
280 + hidden_states = self.input_layernorm(hidden_states)
281 +
282 + hidden_states = self.self_attn(
283 + positions=positions,
284 + hidden_states=hidden_states,
285 + kv_cache=kv_cache,
286 + attn_metadata=attn_metadata,
287 + )
288 + hidden_states = residual + hidden_states
289 +
290 + residual = hidden_states
291 + hidden_states = self.post_attention_layernorm(hidden_states)
292 + hidden_states = self.mlp(hidden_states)
293 + hidden_states = residual + hidden_states
294 + return hidden_states
295 +
296 +
297 +class Phi3SmallModel(nn.Module):
298 +
299 + def __init__(
300 + self,
301 + config: PretrainedConfig,
302 + cache_config: Optional[CacheConfig] = None,
303 + quant_config: Optional[QuantizationConfig] = None,
304 + ):
305 + super().__init__()
306 + self.config = config
307 + self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
308 + config.hidden_size)
309 + self.mup_embedding_multiplier = config.mup_embedding_multiplier
310 + self.layers = nn.ModuleList([
311 + Phi3SmallDecoderLayer(config, layer_idx, cache_config,
312 + quant_config)
313 + for layer_idx in range(config.num_hidden_layers)
314 + ])
315 +
316 + self.final_layernorm = nn.LayerNorm(config.hidden_size,
317 + eps=config.layer_norm_epsilon)
318 +
319 + def get_input_embeddings(self):
320 + return self.embed_tokens
321 +
322 + def set_input_embeddings(self, value):
323 + self.embed_tokens = value
324 +
325 + def forward(
326 + self,
327 + input_ids: torch.LongTensor,
328 + positions: Optional[torch.LongTensor],
329 + kv_caches: List[torch.Tensor],
330 + attn_metadata: AttentionMetadata = None,
331 + ):
332 + hidden_states = self.embed_tokens(input_ids)
333 + if (self.mup_embedding_multiplier is not None
334 + and self.mup_embedding_multiplier > 0.0):
335 + hidden_states = hidden_states * self.mup_embedding_multiplier
336 + for i in range(len(self.layers)):
337 + layer = self.layers[i]
338 + hidden_states = layer(
339 + positions,
340 + hidden_states,
341 + kv_caches[i],
342 + attn_metadata,
343 + )
344 + hidden_states = self.final_layernorm(hidden_states)
345 + return hidden_states
346 +
347 +
348 +class Phi3SmallForCausalLM(nn.Module):
349 + _tied_weights_keys = ["lm_head.weight"]
350 +
351 + def __init__(
352 + self,
353 + config,
354 + cache_config: Optional[CacheConfig] = None,
355 + quant_config: Optional[QuantizationConfig] = None,
356 + lora_config: Optional[LoRAConfig] = None,
357 + ):
358 + super().__init__()
359 + self.config = config
360 + self.quant_config = quant_config
361 + self.model = Phi3SmallModel(config, cache_config, quant_config)
362 + self.vocab_size = config.vocab_size
363 + self.mup_width_multiplier = config.mup_width_multiplier
364 + self.lm_head = ParallelLMHead(
365 + self.vocab_size,
366 + config.hidden_size,
367 + org_num_embeddings=config.vocab_size,
368 + padding_size=DEFAULT_VOCAB_PADDING_SIZE,
369 + )
370 + self.logits_processor = LogitsProcessor(config.vocab_size)
371 + self.sampler = Sampler()
372 +
373 + # tokens in tiktoken but not used
374 + if hasattr(config, 'dummy_token_indices'):
375 + device = self.lm_head.weight.device
376 + self.register_buffer('dummy_token_indices',
377 + torch.LongTensor(
378 + config.dummy_token_indices).to(device),
379 + persistent=False)
380 + else:
381 + self.dummy_token_indices = None
382 +
383 + def get_input_embeddings(self):
384 + return self.model.embed_tokens
385 +
386 + def set_input_embeddings(self, value):
387 + self.model.embed_tokens = value
388 +
389 + def get_output_embeddings(self):
390 + return self.lm_head
391 +
392 + def set_output_embeddings(self, value):
393 + self.lm_head = value
394 +
395 + def set_decoder(self, decoder):
396 + self.model = decoder
397 +
398 + def get_decoder(self):
399 + return self.model
400 +
401 + def compute_logits(self, hidden_states: torch.Tensor,
402 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
403 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
404 + sampling_metadata)
405 + if self.dummy_token_indices is not None and logits is not None:
406 + logits.index_fill_(-1, self.dummy_token_indices, -torch.inf)
407 + return logits
408 +
409 + def forward(
410 + self,
411 + input_ids: torch.LongTensor,
412 + positions: Optional[torch.LongTensor],
413 + kv_caches: List[torch.Tensor],
414 + attn_metadata: AttentionMetadata,
415 + ) -> torch.Tensor:
416 + output_hidden_states = self.model(
417 + input_ids=input_ids,
418 + positions=positions,
419 + kv_caches=kv_caches,
420 + attn_metadata=attn_metadata,
421 + )
422 + output_hidden_states = output_hidden_states
423 + return output_hidden_states
424 +
425 + def sample(
426 + self,
427 + logits: torch.Tensor,
428 + sampling_metadata: SamplingMetadata,
429 + ) -> Optional[SamplerOutput]:
430 +
431 + next_tokens = self.sampler(logits / self.mup_width_multiplier,
432 + sampling_metadata)
433 + return next_tokens
434 +
435 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
436 +
437 + params_dict = dict(self.named_parameters())
438 + for name, loaded_weight in weights:
439 + if "rotary_emb.inv_freq" in name:
440 + continue
441 + if name.endswith(".bias") and name not in params_dict:
442 + continue
443 + param = params_dict[name]
444 + weight_loader = getattr(param, "weight_loader",
445 + default_weight_loader)
446 + weight_loader(param, loaded_weight)
447 + self.lm_head.weight.data.copy_(self.model.embed_tokens.weight.data)
vllm/transformers_utils/config.py
+1 -1
@@ -63,4 +63,4 @@ def get_hf_text_config(config: PretrainedConfig):
63 assert hasattr(config.text_config, "num_attention_heads")
64 return config.text_config
65 else:
66 - return config
66 + return config
\ No newline at end of file