A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel][Backend][Model] Blocksparse flash attention kernel and Phi-3-Small model (#4799)
Co-authored-by: beagleski <yunanzhang@microsoft.com> Co-authored-by: bapatra <bapatra@microsoft.com> Co-authored-by: Barun Patra <codedecde@users.noreply.github.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
Eric Xihui Lin committed
May 25, 2024 at 01:00 UTC
8e192ff967b44b186ea02d30e49fddf656fdfe50
23 files changed
+2446
-88
csrc/attention/attention_kernels.cu
+148
-37
@@ -85,6 +85,7 @@ inline __device__ float block_sum(float* red_smem, float sum) {
85
// Grid: (num_heads, num_seqs, max_num_partitions).
86
template <typename scalar_t, typename cache_t, int HEAD_SIZE, int BLOCK_SIZE,
87
int NUM_THREADS, vllm::Fp8KVCacheDataType KV_DTYPE,
88
+ bool IS_BLOCK_SPARSE,
89
int PARTITION_SIZE = 0> // Zero means no partitioning.
90
__device__ void paged_attention_kernel(
91
float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
@@ -104,7 +105,9 @@ __device__ void paged_attention_kernel(
105
const int max_num_blocks_per_seq,
106
const float* __restrict__ alibi_slopes, // [num_heads]
107
const int q_stride, const int kv_block_stride, const int kv_head_stride,
107
- const float kv_scale) {
108
+ const float kv_scale, const int tp_rank, const int blocksparse_local_blocks,
109
+ const int blocksparse_vert_stride, const int blocksparse_block_size,
110
+ const int blocksparse_head_sliding_step) {
111
const int seq_idx = blockIdx.y;
112
const int partition_idx = blockIdx.z;
113
const int max_num_partitions = gridDim.z;
@@ -202,11 +205,55 @@ __device__ void paged_attention_kernel(
205
// Each thread group in a warp fetches a key from the block, and computes
206
// dot product with the query.
207
const int* block_table = block_tables + seq_idx * max_num_blocks_per_seq;
208
+
209
+ // blocksparse specific vars
210
+ int bs_block_offset;
211
+ int q_bs_block_id;
212
+ if constexpr (IS_BLOCK_SPARSE) {
213
+ // const int num_blocksparse_blocks = DIVIDE_ROUND_UP(seq_len,
214
+ // blocksparse_block_size);
215
+ q_bs_block_id = (seq_len - 1) / blocksparse_block_size;
216
+ if (blocksparse_head_sliding_step >= 0)
217
+ // sliding on q heads
218
+ bs_block_offset =
219
+ (tp_rank * num_heads + head_idx) * blocksparse_head_sliding_step + 1;
220
+ else
221
+ // sliding on kv heads
222
+ bs_block_offset = (tp_rank * num_kv_heads + kv_head_idx) *
223
+ (-blocksparse_head_sliding_step) +
224
+ 1;
225
+ }
226
+
227
for (int block_idx = start_block_idx + warp_idx; block_idx < end_block_idx;
228
block_idx += NUM_WARPS) {
229
// NOTE(woosuk): The block number is stored in int32. However, we cast it to
230
// int64 because int32 can lead to overflow when this variable is multiplied
231
// by large numbers (e.g., kv_block_stride).
232
+ // For blocksparse attention: skip computation on blocks that are not
233
+ // attended
234
+ if constexpr (IS_BLOCK_SPARSE) {
235
+ const int k_bs_block_id = block_idx * BLOCK_SIZE / blocksparse_block_size;
236
+ const bool is_remote =
237
+ ((k_bs_block_id + bs_block_offset) % blocksparse_vert_stride == 0);
238
+ const bool is_local =
239
+ (k_bs_block_id > q_bs_block_id - blocksparse_local_blocks);
240
+ if (!is_remote && !is_local) {
241
+ for (int i = 0; i < NUM_TOKENS_PER_THREAD_GROUP; i++) {
242
+ const int physical_block_offset =
243
+ (thread_group_idx + i * WARP_SIZE) % BLOCK_SIZE;
244
+ const int token_idx = block_idx * BLOCK_SIZE + physical_block_offset;
245
+
246
+ if (thread_group_offset == 0) {
247
+ // NOTE(linxihui): assign very large number to skipped tokens to
248
+ // avoid contribution to the sumexp softmax normalizer. This will
249
+ // not be used at computing sum(softmax*v) as the blocks will be
250
+ // skipped.
251
+ logits[token_idx - start_token_idx] = -FLT_MAX;
252
+ }
253
+ }
254
+ continue;
255
+ }
256
+ }
257
const int64_t physical_block_number =
258
static_cast<int64_t>(block_table[block_idx]);
259
@@ -335,6 +382,15 @@ __device__ void paged_attention_kernel(
382
// NOTE(woosuk): The block number is stored in int32. However, we cast it to
383
// int64 because int32 can lead to overflow when this variable is multiplied
384
// by large numbers (e.g., kv_block_stride).
385
+ // For blocksparse attention: skip computation on blocks that are not
386
+ // attended
387
+ if constexpr (IS_BLOCK_SPARSE) {
388
+ int v_bs_block_id = block_idx * BLOCK_SIZE / blocksparse_block_size;
389
+ if (!((v_bs_block_id + bs_block_offset) % blocksparse_vert_stride == 0) &&
390
+ !((v_bs_block_id > q_bs_block_id - blocksparse_local_blocks))) {
391
+ continue;
392
+ }
393
+ }
394
const int64_t physical_block_number =
395
static_cast<int64_t>(block_table[block_idx]);
396
const int physical_block_offset = (lane % NUM_V_VECS_PER_ROW) * V_VEC_SIZE;
@@ -441,8 +497,8 @@ __device__ void paged_attention_kernel(
497
498
// Grid: (num_heads, num_seqs, 1).
499
template <typename scalar_t, typename cache_t, int HEAD_SIZE, int BLOCK_SIZE,
444
- int NUM_THREADS,
445
- vllm::Fp8KVCacheDataType KV_DTYPE>
500
+ int NUM_THREADS, vllm::Fp8KVCacheDataType KV_DTYPE,
501
+ bool IS_BLOCK_SPARSE>
502
__global__ void paged_attention_v1_kernel(
503
scalar_t* __restrict__ out, // [num_seqs, num_heads, head_size]
504
const scalar_t* __restrict__ q, // [num_seqs, num_heads, head_size]
@@ -457,18 +513,23 @@ __global__ void paged_attention_v1_kernel(
513
const int max_num_blocks_per_seq,
514
const float* __restrict__ alibi_slopes, // [num_heads]
515
const int q_stride, const int kv_block_stride, const int kv_head_stride,
460
- const float kv_scale) {
516
+ const float kv_scale, const int tp_rank, const int blocksparse_local_blocks,
517
+ const int blocksparse_vert_stride, const int blocksparse_block_size,
518
+ const int blocksparse_head_sliding_step) {
519
paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
462
- KV_DTYPE>(
520
+ KV_DTYPE, IS_BLOCK_SPARSE>(
521
/* exp_sums */ nullptr, /* max_logits */ nullptr, out, q, k_cache,
522
v_cache, num_kv_heads, scale, block_tables, seq_lens,
523
max_num_blocks_per_seq, alibi_slopes, q_stride, kv_block_stride,
466
- kv_head_stride, kv_scale);
524
+ kv_head_stride, kv_scale, tp_rank, blocksparse_local_blocks,
525
+ blocksparse_vert_stride, blocksparse_block_size,
526
+ blocksparse_head_sliding_step);
527
}
528
529
// Grid: (num_heads, num_seqs, max_num_partitions).
530
template <typename scalar_t, typename cache_t, int HEAD_SIZE, int BLOCK_SIZE,
531
int NUM_THREADS, vllm::Fp8KVCacheDataType KV_DTYPE,
532
+ bool IS_BLOCK_SPARSE,
533
int PARTITION_SIZE>
534
__global__ void paged_attention_v2_kernel(
535
float* __restrict__ exp_sums, // [num_seqs, num_heads, max_num_partitions]
@@ -488,12 +549,16 @@ __global__ void paged_attention_v2_kernel(
549
const int max_num_blocks_per_seq,
550
const float* __restrict__ alibi_slopes, // [num_heads]
551
const int q_stride, const int kv_block_stride, const int kv_head_stride,
491
- const float kv_scale) {
552
+ const float kv_scale, const int tp_rank, const int blocksparse_local_blocks,
553
+ const int blocksparse_vert_stride, const int blocksparse_block_size,
554
+ const int blocksparse_head_sliding_step) {
555
paged_attention_kernel<scalar_t, cache_t, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS,
493
- KV_DTYPE, PARTITION_SIZE>(
556
+ KV_DTYPE, IS_BLOCK_SPARSE, PARTITION_SIZE>(
557
exp_sums, max_logits, tmp_out, q, k_cache, v_cache, num_kv_heads, scale,
558
block_tables, seq_lens, max_num_blocks_per_seq, alibi_slopes, q_stride,
496
- kv_block_stride, kv_head_stride, kv_scale);
559
+ kv_block_stride, kv_head_stride, kv_scale, tp_rank,
560
+ blocksparse_local_blocks, blocksparse_vert_stride, blocksparse_block_size,
561
+ blocksparse_head_sliding_step);
562
}
563
564
// Grid: (num_heads, num_seqs).
@@ -607,25 +672,32 @@ __global__ void paged_attention_v2_reduce_kernel(
672
673
#define LAUNCH_PAGED_ATTENTION_V1(HEAD_SIZE) \
674
VLLM_DevFuncAttribute_SET_MaxDynamicSharedMemorySize( \
610
- ((void*)vllm::paged_attention_v1_kernel< \
611
- T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, NUM_THREADS, KV_DTYPE>), \
675
+ ((void*)vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, \
676
+ BLOCK_SIZE, NUM_THREADS, \
677
+ KV_DTYPE, IS_BLOCK_SPARSE>), \
678
shared_mem_size); \
679
vllm::paged_attention_v1_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
614
- NUM_THREADS, KV_DTYPE> \
680
+ NUM_THREADS, KV_DTYPE, IS_BLOCK_SPARSE> \
681
<<<grid, block, shared_mem_size, stream>>>( \
682
out_ptr, query_ptr, key_cache_ptr, value_cache_ptr, num_kv_heads, \
683
scale, block_tables_ptr, seq_lens_ptr, max_num_blocks_per_seq, \
684
alibi_slopes_ptr, q_stride, kv_block_stride, kv_head_stride, \
619
- kv_scale);
685
+ kv_scale, tp_rank, blocksparse_local_blocks, \
686
+ blocksparse_vert_stride, blocksparse_block_size, \
687
+ blocksparse_head_sliding_step);
688
689
// TODO(woosuk): Tune NUM_THREADS.
690
template <typename T, typename CACHE_T, int BLOCK_SIZE,
623
- vllm::Fp8KVCacheDataType KV_DTYPE, int NUM_THREADS = 128>
691
+ vllm::Fp8KVCacheDataType KV_DTYPE, bool IS_BLOCK_SPARSE,
692
+ int NUM_THREADS = 128>
693
void paged_attention_v1_launcher(
694
torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
695
torch::Tensor& value_cache, int num_kv_heads, float scale,
696
torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
628
- const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale) {
697
+ const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale,
698
+ const int tp_rank, const int blocksparse_local_blocks,
699
+ const int blocksparse_vert_stride, const int blocksparse_block_size,
700
+ const int blocksparse_head_sliding_step) {
701
int num_seqs = query.size(0);
702
int num_heads = query.size(1);
703
int head_size = query.size(2);
@@ -691,23 +763,36 @@ void paged_attention_v1_launcher(
763
}
764
}
765
694
-#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
695
- paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
766
+#define CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE, IS_BLOCK_SPARSE) \
767
+ paged_attention_v1_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
768
+ IS_BLOCK_SPARSE>( \
769
out, query, key_cache, value_cache, num_kv_heads, scale, block_tables, \
697
- seq_lens, max_seq_len, alibi_slopes, kv_scale);
770
+ seq_lens, max_seq_len, alibi_slopes, kv_scale, tp_rank, \
771
+ blocksparse_local_blocks, blocksparse_vert_stride, \
772
+ blocksparse_block_size, blocksparse_head_sliding_step);
773
+
774
+#define CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
775
+ switch (is_block_sparse) { \
776
+ case true: \
777
+ CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, true); \
778
+ break; \
779
+ case false: \
780
+ CALL_V1_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, false); \
781
+ break; \
782
+ }
783
784
// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
785
// 1, 2, 4, 64, 128, 256.
786
#define CALL_V1_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
787
switch (block_size) { \
788
case 8: \
704
- CALL_V1_LAUNCHER(T, CACHE_T, 8, KV_DTYPE); \
789
+ CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 8, KV_DTYPE); \
790
break; \
791
case 16: \
707
- CALL_V1_LAUNCHER(T, CACHE_T, 16, KV_DTYPE); \
792
+ CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 16, KV_DTYPE); \
793
break; \
794
case 32: \
710
- CALL_V1_LAUNCHER(T, CACHE_T, 32, KV_DTYPE); \
795
+ CALL_V1_LAUNCHER_SPARSITY(T, CACHE_T, 32, KV_DTYPE); \
796
break; \
797
default: \
798
TORCH_CHECK(false, "Unsupported block size: ", block_size); \
@@ -727,18 +812,26 @@ void paged_attention_v1(
812
torch::Tensor& seq_lens, // [num_seqs]
813
int block_size, int max_seq_len,
814
const c10::optional<torch::Tensor>& alibi_slopes,
730
- const std::string& kv_cache_dtype, float kv_scale){
815
+ const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
816
+ const int blocksparse_local_blocks, const int blocksparse_vert_stride,
817
+ const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
818
+ const bool is_block_sparse = (blocksparse_vert_stride > 1);
819
+
820
+ DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
821
+ CALL_V1_LAUNCHER_BLOCK_SIZE)
822
+}
823
732
- DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
733
- CALL_V1_LAUNCHER_BLOCK_SIZE)}
824
#define LAUNCH_PAGED_ATTENTION_V2(HEAD_SIZE) \
825
vllm::paged_attention_v2_kernel<T, CACHE_T, HEAD_SIZE, BLOCK_SIZE, \
736
- NUM_THREADS, KV_DTYPE, PARTITION_SIZE> \
826
+ NUM_THREADS, KV_DTYPE, IS_BLOCK_SPARSE, \
827
+ PARTITION_SIZE> \
828
<<<grid, block, shared_mem_size, stream>>>( \
829
exp_sums_ptr, max_logits_ptr, tmp_out_ptr, query_ptr, key_cache_ptr, \
830
value_cache_ptr, num_kv_heads, scale, block_tables_ptr, \
831
seq_lens_ptr, max_num_blocks_per_seq, alibi_slopes_ptr, q_stride, \
741
- kv_block_stride, kv_head_stride, kv_scale); \
832
+ kv_block_stride, kv_head_stride, kv_scale, tp_rank, \
833
+ blocksparse_local_blocks, blocksparse_vert_stride, \
834
+ blocksparse_block_size, blocksparse_head_sliding_step); \
835
vllm::paged_attention_v2_reduce_kernel<T, HEAD_SIZE, NUM_THREADS, \
836
PARTITION_SIZE> \
837
<<<reduce_grid, block, reduce_shared_mem_size, stream>>>( \
@@ -746,14 +839,17 @@ void paged_attention_v1(
839
max_num_partitions);
840
841
template <typename T, typename CACHE_T, int BLOCK_SIZE,
749
- vllm::Fp8KVCacheDataType KV_DTYPE, int NUM_THREADS = 128,
750
- int PARTITION_SIZE = 512>
842
+ vllm::Fp8KVCacheDataType KV_DTYPE, bool IS_BLOCK_SPARSE,
843
+ int NUM_THREADS = 128, int PARTITION_SIZE = 512>
844
void paged_attention_v2_launcher(
845
torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
846
torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
847
torch::Tensor& value_cache, int num_kv_heads, float scale,
848
torch::Tensor& block_tables, torch::Tensor& seq_lens, int max_seq_len,
756
- const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale) {
849
+ const c10::optional<torch::Tensor>& alibi_slopes, float kv_scale,
850
+ const int tp_rank, const int blocksparse_local_blocks,
851
+ const int blocksparse_vert_stride, const int blocksparse_block_size,
852
+ const int blocksparse_head_sliding_step) {
853
int num_seqs = query.size(0);
854
int num_heads = query.size(1);
855
int head_size = query.size(2);
@@ -824,24 +920,36 @@ void paged_attention_v2_launcher(
920
}
921
}
922
827
-#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE) \
828
- paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE>( \
923
+#define CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, KV_DTYPE, IS_BLOCK_SPARSE) \
924
+ paged_attention_v2_launcher<T, CACHE_T, BLOCK_SIZE, KV_DTYPE, \
925
+ IS_BLOCK_SPARSE>( \
926
out, exp_sums, max_logits, tmp_out, query, key_cache, value_cache, \
927
num_kv_heads, scale, block_tables, seq_lens, max_seq_len, alibi_slopes, \
831
- kv_scale);
928
+ kv_scale, tp_rank, blocksparse_local_blocks, blocksparse_vert_stride, \
929
+ blocksparse_block_size, blocksparse_head_sliding_step);
930
+
931
+#define CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE) \
932
+ switch (is_block_sparse) { \
933
+ case true: \
934
+ CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, true); \
935
+ break; \
936
+ case false: \
937
+ CALL_V2_LAUNCHER(T, CACHE_T, BLOCK_SIZE, IS_FP8_KV_CACHE, false); \
938
+ break; \
939
+ }
940
941
// NOTE(woosuk): To reduce the compilation time, we omitted block sizes
942
// 1, 2, 4, 64, 128, 256.
943
#define CALL_V2_LAUNCHER_BLOCK_SIZE(T, CACHE_T, KV_DTYPE) \
944
switch (block_size) { \
945
case 8: \
838
- CALL_V2_LAUNCHER(T, CACHE_T, 8, KV_DTYPE); \
946
+ CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 8, KV_DTYPE); \
947
break; \
948
case 16: \
841
- CALL_V2_LAUNCHER(T, CACHE_T, 16, KV_DTYPE); \
949
+ CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 16, KV_DTYPE); \
950
break; \
951
case 32: \
844
- CALL_V2_LAUNCHER(T, CACHE_T, 32, KV_DTYPE); \
952
+ CALL_V2_LAUNCHER_SPARSITY(T, CACHE_T, 32, KV_DTYPE); \
953
break; \
954
default: \
955
TORCH_CHECK(false, "Unsupported block size: ", block_size); \
@@ -865,7 +973,10 @@ void paged_attention_v2(
973
torch::Tensor& seq_lens, // [num_seqs]
974
int block_size, int max_seq_len,
975
const c10::optional<torch::Tensor>& alibi_slopes,
868
- const std::string& kv_cache_dtype, float kv_scale) {
976
+ const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
977
+ const int blocksparse_local_blocks, const int blocksparse_vert_stride,
978
+ const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
979
+ const bool is_block_sparse = (blocksparse_vert_stride > 1);
980
DISPATCH_BY_KV_CACHE_DTYPE(query.dtype(), kv_cache_dtype,
981
CALL_V2_LAUNCHER_BLOCK_SIZE)
982
}
@@ -873,4 +984,4 @@ void paged_attention_v2(
984
#undef WARP_SIZE
985
#undef MAX
986
#undef MIN
876
-#undef DIVIDE_ROUND_UP
987
+#undef DIVIDE_ROUND_UP
\ No newline at end of file
csrc/cpu/attention.cpp
+21
-16
@@ -415,14 +415,17 @@ void paged_attention_v1_impl_launcher(
415
}
416
} // namespace
417
418
-void paged_attention_v1(torch::Tensor& out, torch::Tensor& query,
419
- torch::Tensor& key_cache, torch::Tensor& value_cache,
420
- int num_kv_heads, float scale,
421
- torch::Tensor& block_tables, torch::Tensor& seq_lens,
422
- int block_size, int max_seq_len,
423
- const c10::optional<torch::Tensor>& alibi_slopes,
424
- const std::string& kv_cache_dtype, float kv_scale) {
418
+void paged_attention_v1(
419
+ torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
420
+ torch::Tensor& value_cache, int num_kv_heads, float scale,
421
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
422
+ int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
423
+ const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
424
+ const int blocksparse_local_blocks, const int blocksparse_vert_stride,
425
+ const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
426
TORCH_CHECK(kv_scale == 1.0f);
427
+ TORCH_CHECK(blocksparse_vert_stride <= 1,
428
+ "CPU backend does not support blocksparse attention yet.");
429
VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v1_impl",
430
[&] {
431
CPU_KERNEL_GUARD_IN(paged_attention_v1_impl)
@@ -726,16 +729,18 @@ void paged_attention_v2_impl_launcher(
729
}
730
} // namespace
731
729
-void paged_attention_v2(torch::Tensor& out, torch::Tensor& exp_sums,
730
- torch::Tensor& max_logits, torch::Tensor& tmp_out,
731
- torch::Tensor& query, torch::Tensor& key_cache,
732
- torch::Tensor& value_cache, int num_kv_heads,
733
- float scale, torch::Tensor& block_tables,
734
- torch::Tensor& seq_lens, int block_size,
735
- int max_seq_len,
736
- const c10::optional<torch::Tensor>& alibi_slopes,
737
- const std::string& kv_cache_dtype, float kv_scale) {
732
+void paged_attention_v2(
733
+ torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
734
+ torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
735
+ torch::Tensor& value_cache, int num_kv_heads, float scale,
736
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
737
+ int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
738
+ const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
739
+ const int blocksparse_local_blocks, const int blocksparse_vert_stride,
740
+ const int blocksparse_block_size, const int blocksparse_head_sliding_step) {
741
TORCH_CHECK(kv_scale == 1.0f);
742
+ TORCH_CHECK(blocksparse_vert_stride <= 1,
743
+ "CPU backend does not support blocksparse attention yet.");
744
VLLM_DISPATCH_FLOATING_TYPES(query.scalar_type(), "paged_attention_v2_impl",
745
[&] {
746
CPU_KERNEL_GUARD_IN(paged_attention_v2_impl)
csrc/ops.h
+18
-17
@@ -2,23 +2,24 @@
2
3
#include <torch/extension.h>
4
5
-void paged_attention_v1(torch::Tensor& out, torch::Tensor& query,
6
- torch::Tensor& key_cache, torch::Tensor& value_cache,
7
- int num_kv_heads, float scale,
8
- torch::Tensor& block_tables, torch::Tensor& seq_lens,
9
- int block_size, int max_seq_len,
10
- const c10::optional<torch::Tensor>& alibi_slopes,
11
- const std::string& kv_cache_dtype, float kv_scale);
12
-
13
-void paged_attention_v2(torch::Tensor& out, torch::Tensor& exp_sums,
14
- torch::Tensor& max_logits, torch::Tensor& tmp_out,
15
- torch::Tensor& query, torch::Tensor& key_cache,
16
- torch::Tensor& value_cache, int num_kv_heads,
17
- float scale, torch::Tensor& block_tables,
18
- torch::Tensor& seq_lens, int block_size,
19
- int max_seq_len,
20
- const c10::optional<torch::Tensor>& alibi_slopes,
21
- const std::string& kv_cache_dtype, float kv_scale);
5
+void paged_attention_v1(
6
+ torch::Tensor& out, torch::Tensor& query, torch::Tensor& key_cache,
7
+ torch::Tensor& value_cache, int num_kv_heads, float scale,
8
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
9
+ int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
10
+ const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
11
+ const int blocksparse_local_blocks, const int blocksparse_vert_stride,
12
+ const int blocksparse_block_size, const int blocksparse_head_sliding_step);
13
+
14
+void paged_attention_v2(
15
+ torch::Tensor& out, torch::Tensor& exp_sums, torch::Tensor& max_logits,
16
+ torch::Tensor& tmp_out, torch::Tensor& query, torch::Tensor& key_cache,
17
+ torch::Tensor& value_cache, int num_kv_heads, float scale,
18
+ torch::Tensor& block_tables, torch::Tensor& seq_lens, int block_size,
19
+ int max_seq_len, const c10::optional<torch::Tensor>& alibi_slopes,
20
+ const std::string& kv_cache_dtype, float kv_scale, const int tp_rank,
21
+ const int blocksparse_local_blocks, const int blocksparse_vert_stride,
22
+ const int blocksparse_block_size, const int blocksparse_head_sliding_step);
23
24
void rms_norm(torch::Tensor& out, torch::Tensor& input, torch::Tensor& weight,
25
float epsilon);
docs/source/models/supported_models.rst
+4
@@ -123,6 +123,10 @@ Alongside each architecture, we include some popular models that use it.
123
- Phi-3
124
- :code:`microsoft/Phi-3-mini-4k-instruct`, :code:`microsoft/Phi-3-mini-128k-instruct`, etc.
125
-
126
+ * - :code:`Phi3SmallForCausalLM`
127
+ - Phi-3-Small
128
+ - :code:`microsoft/Phi-3-small-8k-instruct`, :code:`microsoft/Phi-3-small-128k-instruct`, etc.
129
+ -
130
* - :code:`QWenLMHeadModel`
131
- Qwen
132
- :code:`Qwen/Qwen-7B`, :code:`Qwen/Qwen-7B-Chat`, etc.
tests/kernels/test_blocksparse_attention.py
new
+442
@@ -0,0 +1,442 @@
1
+import random
2
+from typing import List, Optional, Tuple
3
+
4
+import pytest
5
+import torch
6
+
7
+from vllm import _custom_ops as ops
8
+from vllm.attention.ops.blocksparse_attention.interface import (
9
+ LocalStridedBlockSparseAttn)
10
+from vllm.utils import get_max_shared_memory_bytes, is_hip
11
+
12
+from .allclose_default import get_default_atol, get_default_rtol
13
+
14
+FLOAT32_BYTES = torch.finfo(torch.float).bits // 8
15
+# This will change depending on the compute capability.
16
+# - 512 as a buffer
17
+MAX_SEQ_LEN = get_max_shared_memory_bytes() // FLOAT32_BYTES - 512
18
+# MAX_SEQ_LEN = 2771
19
+
20
+# There may not be enough gpu memory due to large NUM_BLOCKS.
21
+# Reduce NUM_BLOCKS when it happens.
22
+NUM_BLOCKS = 4321 # Arbitrary values for testing
23
+PARTITION_SIZE = 512
24
+DTYPES = [torch.half, torch.bfloat16]
25
+NUM_GEN_SEQS = [3] # Arbitrary values for testing
26
+NUM_PREFILL_SEQS = [3] # Arbitrary values for testing
27
+NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
28
+
29
+HEAD_SIZES = [64, 112]
30
+BLOCK_SIZES = [16, 32]
31
+USE_ALIBI = [False, True]
32
+KV_CACHE_DTYPE = ["auto", "fp8"]
33
+SEEDS = [0]
34
+CUDA_DEVICES = ['cuda:0']
35
+BLOCKSPARSE_LOCAL_BLOCKS = [16]
36
+BLOCKSPARSE_VERT_STRIDES = [8]
37
+
38
+BLOCKSPARSE_BLOCK_SIZES = [64]
39
+BLOCKSPARSE_HEADS_SLIDINGS = [0, 2, -1]
40
+BLOCKSPARSE_HOMO_HEADS = [True, False]
41
+
42
+
43
+def ref_masked_attention(
44
+ query: torch.Tensor,
45
+ key: torch.Tensor,
46
+ value: torch.Tensor,
47
+ scale: float,
48
+ attn_mask: Optional[torch.Tensor] = None,
49
+) -> torch.Tensor:
50
+ attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
51
+ if attn_mask is not None:
52
+ attn_weights = attn_weights + attn_mask.float()
53
+ attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
54
+ out = torch.einsum("hqk,khd->qhd", attn_weights, value)
55
+ return out
56
+
57
+
58
+def ref_single_query_cached_kv_attention(
59
+ output: torch.Tensor,
60
+ query: torch.Tensor,
61
+ num_queries_per_kv: int,
62
+ key_cache: torch.Tensor,
63
+ value_cache: torch.Tensor,
64
+ block_tables: torch.Tensor,
65
+ seq_lens: torch.Tensor,
66
+ scale: float,
67
+ alibi_slopes: Optional[torch.Tensor],
68
+ tp_rank: int = 0,
69
+ blocksparse_local_blocks: int = 0,
70
+ blocksparse_vert_stride: int = 1,
71
+ blocksparse_block_size: int = 64,
72
+ blocksparse_head_sliding_step: int = 0,
73
+) -> None:
74
+ num_query_heads = query.shape[1]
75
+ num_kv_heads = value_cache.shape[1]
76
+ head_size = value_cache.shape[2]
77
+ block_size = value_cache.shape[3]
78
+ num_seqs = query.shape[0]
79
+
80
+ block_tables = block_tables.cpu().tolist()
81
+ seq_lens = seq_lens.cpu().tolist()
82
+ for i in range(num_seqs):
83
+ q = query[i].unsqueeze(0)
84
+ block_table = block_tables[i]
85
+ seq_len = int(seq_lens[i])
86
+
87
+ keys = []
88
+ values = []
89
+ for j in range(seq_len):
90
+ block_number = int(block_table[j // block_size])
91
+ block_offset = j % block_size
92
+
93
+ k = key_cache[block_number, :, :, block_offset, :]
94
+ k = k.reshape(num_kv_heads, head_size)
95
+ keys.append(k)
96
+
97
+ v = value_cache[block_number, :, :, block_offset]
98
+ values.append(v)
99
+ keys = torch.stack(keys, dim=0)
100
+ values = torch.stack(values, dim=0)
101
+ if num_queries_per_kv > 1:
102
+ # Handle MQA and GQA
103
+ keys = torch.repeat_interleave(keys, num_queries_per_kv, dim=1)
104
+ values = torch.repeat_interleave(values, num_queries_per_kv, dim=1)
105
+
106
+ alibi_bias = None
107
+ if alibi_slopes is not None:
108
+ # Create the ALiBi bias used in the paged attention kernel.
109
+ position_ids = torch.arange(seq_len).int()
110
+ alibi_bias = (position_ids - seq_len + 1).float()
111
+ alibi_bias = alibi_slopes.view(-1, 1, 1) * alibi_bias.view(
112
+ 1, 1, -1)
113
+
114
+ if blocksparse_vert_stride >= 1:
115
+ bsize = blocksparse_block_size
116
+ hsliding = blocksparse_head_sliding_step
117
+ vert = blocksparse_vert_stride
118
+ locals = blocksparse_local_blocks
119
+ qb = (seq_len - 1) // bsize
120
+ attn_mask = q.new_zeros(
121
+ (num_query_heads, 1, seq_len)).float() - torch.inf
122
+ for h in range(num_query_heads):
123
+ if hsliding >= 0: # slide with q heads
124
+ bs_offset = (tp_rank * num_query_heads + h) * hsliding + 1
125
+ else: # slide with kv heads
126
+ bs_offset = (tp_rank * num_kv_heads +
127
+ h // num_queries_per_kv) * (-hsliding) + 1
128
+ for kb in range(qb + 1):
129
+ kj = kb * bsize
130
+ if (qb - kb) < locals or \
131
+ (kb + bs_offset) % vert == 0:
132
+ attn_mask[h, 0, kj:min(kj + bsize, seq_len)] = 0
133
+ if alibi_bias is not None:
134
+ attn_mask += alibi_bias
135
+ else:
136
+ attn_mask = alibi_bias
137
+
138
+ out = ref_masked_attention(q, keys, values, scale, attn_mask=attn_mask)
139
+ out = out.view(num_query_heads, head_size)
140
+ output[i].copy_(out, non_blocking=True)
141
+
142
+
143
+@pytest.mark.parametrize("version", ["v1", "v2"])
144
+@pytest.mark.parametrize("num_seqs", NUM_GEN_SEQS)
145
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
146
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
147
+@pytest.mark.parametrize("use_alibi", USE_ALIBI)
148
+@pytest.mark.parametrize("block_size", BLOCK_SIZES)
149
+@pytest.mark.parametrize("dtype", DTYPES)
150
+@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
151
+@pytest.mark.parametrize("seed", SEEDS)
152
+@pytest.mark.parametrize("device", CUDA_DEVICES)
153
+@pytest.mark.parametrize("blocksparse_local_blocks", BLOCKSPARSE_LOCAL_BLOCKS)
154
+@pytest.mark.parametrize("blocksparse_vert_stride", BLOCKSPARSE_VERT_STRIDES)
155
+@pytest.mark.parametrize("blocksparse_block_size", BLOCKSPARSE_BLOCK_SIZES)
156
+@pytest.mark.parametrize("blocksparse_head_sliding_step",
157
+ BLOCKSPARSE_HEADS_SLIDINGS)
158
+def test_paged_attention(
159
+ kv_cache_factory,
160
+ version: str,
161
+ num_seqs: int,
162
+ num_heads: Tuple[int, int],
163
+ head_size: int,
164
+ use_alibi: bool,
165
+ block_size: int,
166
+ dtype: torch.dtype,
167
+ kv_cache_dtype: str,
168
+ seed: int,
169
+ device: str,
170
+ blocksparse_local_blocks: int,
171
+ blocksparse_vert_stride: int,
172
+ blocksparse_block_size: int,
173
+ blocksparse_head_sliding_step: int,
174
+) -> None:
175
+ random.seed(seed)
176
+ torch.random.manual_seed(seed)
177
+ if torch.cuda.is_available():
178
+ torch.cuda.manual_seed(seed)
179
+ torch.set_default_device(device)
180
+ scale = float(1.0 / (head_size**0.5))
181
+ num_query_heads, num_kv_heads = num_heads
182
+ query = torch.empty(num_seqs, num_query_heads, head_size, dtype=dtype)
183
+ query.uniform_(-scale, scale)
184
+
185
+ assert num_query_heads % num_kv_heads == 0
186
+ num_queries_per_kv = num_query_heads // num_kv_heads
187
+ alibi_slopes = None
188
+ if use_alibi:
189
+ alibi_slopes = torch.rand(num_query_heads, dtype=torch.float)
190
+
191
+ seq_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_seqs)]
192
+ seq_lens[-1] = MAX_SEQ_LEN
193
+ max_seq_len = max(seq_lens)
194
+ seq_lens = torch.tensor(seq_lens, dtype=torch.int)
195
+
196
+ # Create the block tables.
197
+ max_num_blocks_per_seq = (max_seq_len + block_size - 1) // block_size
198
+ block_tables = []
199
+ for _ in range(num_seqs):
200
+ block_table = [
201
+ random.randint(0, NUM_BLOCKS - 1)
202
+ for _ in range(max_num_blocks_per_seq)
203
+ ]
204
+ block_tables.append(block_table)
205
+ block_tables = torch.tensor(block_tables, dtype=torch.int)
206
+
207
+ # Create the KV caches.
208
+ key_caches, value_caches = kv_cache_factory(NUM_BLOCKS, block_size, 1,
209
+ num_kv_heads, head_size,
210
+ kv_cache_dtype, dtype, seed,
211
+ device)
212
+ key_cache, value_cache = key_caches[0], value_caches[0]
213
+
214
+ # Using default kv_scale
215
+ kv_scale = 1.0
216
+ tp_rank = 0
217
+
218
+ # Call the paged attention kernel.
219
+ output = torch.empty_like(query)
220
+ if version == "v1":
221
+ ops.paged_attention_v1(
222
+ output,
223
+ query,
224
+ key_cache,
225
+ value_cache,
226
+ num_kv_heads,
227
+ scale,
228
+ block_tables,
229
+ seq_lens,
230
+ block_size,
231
+ max_seq_len,
232
+ alibi_slopes,
233
+ kv_cache_dtype,
234
+ kv_scale,
235
+ tp_rank=tp_rank,
236
+ blocksparse_local_blocks=blocksparse_local_blocks,
237
+ blocksparse_vert_stride=blocksparse_vert_stride,
238
+ blocksparse_block_size=blocksparse_block_size,
239
+ blocksparse_head_sliding_step=blocksparse_head_sliding_step,
240
+ )
241
+ elif version == "v2":
242
+ num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
243
+ assert PARTITION_SIZE % block_size == 0
244
+ num_seqs, num_heads, head_size = output.shape
245
+ tmp_output = torch.empty(
246
+ size=(num_seqs, num_heads, num_partitions, head_size),
247
+ dtype=output.dtype,
248
+ )
249
+ exp_sums = torch.empty(
250
+ size=(num_seqs, num_heads, num_partitions),
251
+ dtype=torch.float32,
252
+ )
253
+ max_logits = torch.empty_like(exp_sums)
254
+ ops.paged_attention_v2(
255
+ output,
256
+ exp_sums,
257
+ max_logits,
258
+ tmp_output,
259
+ query,
260
+ key_cache,
261
+ value_cache,
262
+ num_kv_heads,
263
+ scale,
264
+ block_tables,
265
+ seq_lens,
266
+ block_size,
267
+ max_seq_len,
268
+ alibi_slopes,
269
+ kv_cache_dtype,
270
+ kv_scale,
271
+ tp_rank=tp_rank,
272
+ blocksparse_local_blocks=blocksparse_local_blocks,
273
+ blocksparse_vert_stride=blocksparse_vert_stride,
274
+ blocksparse_block_size=blocksparse_block_size,
275
+ blocksparse_head_sliding_step=blocksparse_head_sliding_step,
276
+ )
277
+ else:
278
+ raise AssertionError(f"Unknown version: {version}")
279
+
280
+ # Run the reference implementation.
281
+ if kv_cache_dtype == "fp8":
282
+ # Convert cache data back to dtype.
283
+ x = 16 // torch.tensor([], dtype=dtype).element_size()
284
+ key_cache_shape = (NUM_BLOCKS, num_kv_heads, head_size // x,
285
+ block_size, x)
286
+ dequantized_key_cache = torch.empty(size=key_cache_shape,
287
+ dtype=dtype,
288
+ device=device)
289
+ ops.convert_fp8(dequantized_key_cache, key_cache)
290
+ key_cache = dequantized_key_cache
291
+
292
+ value_cache_shape = value_cache.shape
293
+ dequantized_value_cache = torch.empty(size=value_cache_shape,
294
+ dtype=dtype,
295
+ device=device)
296
+ ops.convert_fp8(dequantized_value_cache, value_cache)
297
+ value_cache = dequantized_value_cache
298
+
299
+ ref_output = torch.empty_like(query)
300
+ ref_single_query_cached_kv_attention(
301
+ ref_output,
302
+ query,
303
+ num_queries_per_kv,
304
+ key_cache,
305
+ value_cache,
306
+ block_tables,
307
+ seq_lens,
308
+ scale,
309
+ alibi_slopes,
310
+ tp_rank,
311
+ blocksparse_local_blocks,
312
+ blocksparse_vert_stride,
313
+ blocksparse_block_size,
314
+ blocksparse_head_sliding_step,
315
+ )
316
+
317
+ # NOTE(woosuk): Due to the kernel-level differences in the two
318
+ # implementations, there is a small numerical difference in the two
319
+ # outputs. Thus, we use a relaxed tolerance for the test.
320
+ atol = get_default_atol(output) if is_hip() else 1e-3
321
+ rtol = get_default_rtol(output) if is_hip() else 1e-5
322
+
323
+ # NOTE(zhaoyang): FP8 KV Cache will introduce quantization error,
324
+ # so we use a relaxed tolerance for the test.
325
+ atol, rtol = 1e-3, 1e-5
326
+ if kv_cache_dtype == "fp8":
327
+ atol, rtol = 1e-2, 1e-5
328
+ assert torch.allclose(output, ref_output, atol=atol, rtol=rtol)
329
+
330
+
331
+def ref_multi_query_kv_attention(
332
+ cu_seq_lens: List[int],
333
+ query: torch.Tensor,
334
+ key: torch.Tensor,
335
+ value: torch.Tensor,
336
+ scale: float,
337
+ dtype: torch.dtype,
338
+) -> torch.Tensor:
339
+ num_seqs = len(cu_seq_lens) - 1
340
+ ref_outputs = []
341
+ for i in range(num_seqs):
342
+ start_idx = cu_seq_lens[i]
343
+ end_idx = cu_seq_lens[i + 1]
344
+ seq_len = end_idx - start_idx
345
+
346
+ # Create attention mask.
347
+ attn_mask = torch.triu(torch.ones(seq_len, seq_len, dtype=dtype),
348
+ diagonal=1)
349
+ attn_mask = attn_mask * torch.finfo(dtype).min
350
+ attn_mask = attn_mask.to(dtype=dtype)
351
+
352
+ ref_output = ref_masked_attention(
353
+ query[start_idx:end_idx],
354
+ key[start_idx:end_idx],
355
+ value[start_idx:end_idx],
356
+ scale,
357
+ attn_mask=attn_mask,
358
+ )
359
+ ref_outputs.append(ref_output)
360
+ ref_output = torch.cat(ref_outputs, dim=0)
361
+ return ref_output
362
+
363
+
364
+@pytest.mark.parametrize("num_seqs", NUM_PREFILL_SEQS)
365
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
366
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
367
+@pytest.mark.parametrize("blocksparse_local_blocks", BLOCKSPARSE_LOCAL_BLOCKS)
368
+@pytest.mark.parametrize("blocksparse_vert_stride", BLOCKSPARSE_VERT_STRIDES)
369
+@pytest.mark.parametrize("blocksparse_block_size", BLOCKSPARSE_BLOCK_SIZES)
370
+@pytest.mark.parametrize("blocksparse_homo_heads", BLOCKSPARSE_HOMO_HEADS)
371
+@pytest.mark.parametrize("dtype", DTYPES)
372
+@pytest.mark.parametrize("seed", SEEDS)
373
+@pytest.mark.parametrize("device", CUDA_DEVICES)
374
+@torch.inference_mode()
375
+def test_varlen_blocksparse_attention_prefill(
376
+ num_seqs: int,
377
+ num_heads: Tuple[int, int],
378
+ head_size: int,
379
+ blocksparse_local_blocks: int,
380
+ blocksparse_vert_stride: int,
381
+ blocksparse_block_size: int,
382
+ blocksparse_homo_heads: bool,
383
+ dtype: torch.dtype,
384
+ seed: int,
385
+ device: str,
386
+) -> None:
387
+ random.seed(seed)
388
+ torch.random.manual_seed(seed)
389
+ if torch.cuda.is_available():
390
+ torch.cuda.manual_seed(seed)
391
+ torch.set_default_device(device)
392
+ # MAX_SEQ_LEN sometimes causes OOM in the reference implementation.
393
+ # As the xformers library is already tested with its own tests, we can use
394
+ # a smaller MAX_SEQ_LEN here.
395
+ max_len = min(MAX_SEQ_LEN, 4096)
396
+ seq_lens = random.sample(range(1, max_len), num_seqs)
397
+ cu_seq_lens = torch.cumsum(torch.tensor([0] + seq_lens), dim=0)
398
+ num_tokens = sum(seq_lens)
399
+
400
+ scale = float(1.0 / (head_size**0.5))
401
+ num_query_heads, num_kv_heads = num_heads
402
+ assert num_query_heads % num_kv_heads == 0
403
+ num_queries_per_kv = num_query_heads // num_kv_heads
404
+
405
+ qkv = torch.empty(num_tokens,
406
+ num_query_heads + 2 * num_kv_heads,
407
+ head_size,
408
+ dtype=dtype)
409
+ qkv.uniform_(-scale, scale)
410
+ query, key, value = qkv.split(
411
+ [num_query_heads, num_kv_heads, num_kv_heads], dim=1)
412
+
413
+ bs_attn_op = LocalStridedBlockSparseAttn(
414
+ num_query_heads,
415
+ max_len,
416
+ local_blocks=blocksparse_local_blocks,
417
+ vert_stride=blocksparse_vert_stride,
418
+ block_size=blocksparse_block_size,
419
+ device=device,
420
+ dtype=dtype,
421
+ homo_head=blocksparse_homo_heads)
422
+
423
+ output = bs_attn_op(query,
424
+ key,
425
+ value,
426
+ cu_seq_lens.to(device),
427
+ sm_scale=scale)
428
+
429
+ if num_queries_per_kv > 1:
430
+ # Handle MQA and GQA
431
+ key = torch.repeat_interleave(key, num_queries_per_kv, dim=1)
432
+ value = torch.repeat_interleave(value, num_queries_per_kv, dim=1)
433
+
434
+ ref_output = ref_multi_query_kv_attention(
435
+ cu_seq_lens,
436
+ query,
437
+ key,
438
+ value,
439
+ scale,
440
+ dtype,
441
+ )
442
+ assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2)
vllm/_custom_ops.py
+21
-9
@@ -45,11 +45,17 @@ def paged_attention_v1(
45
alibi_slopes: Optional[torch.Tensor],
46
kv_cache_dtype: str,
47
kv_scale: float,
48
+ tp_rank: int = 0,
49
+ blocksparse_local_blocks: int = 0,
50
+ blocksparse_vert_stride: int = 0,
51
+ blocksparse_block_size: int = 64,
52
+ blocksparse_head_sliding_step: int = 0,
53
) -> None:
49
- vllm_ops.paged_attention_v1(out, query, key_cache, value_cache,
50
- num_kv_heads, scale, block_tables, seq_lens,
51
- block_size, max_seq_len, alibi_slopes,
52
- kv_cache_dtype, kv_scale)
54
+ vllm_ops.paged_attention_v1(
55
+ out, query, key_cache, value_cache, num_kv_heads, scale, block_tables,
56
+ seq_lens, block_size, max_seq_len, alibi_slopes, kv_cache_dtype,
57
+ kv_scale, tp_rank, blocksparse_local_blocks, blocksparse_vert_stride,
58
+ blocksparse_block_size, blocksparse_head_sliding_step)
59
60
61
def paged_attention_v2(
@@ -69,12 +75,18 @@ def paged_attention_v2(
75
alibi_slopes: Optional[torch.Tensor],
76
kv_cache_dtype: str,
77
kv_scale: float,
78
+ tp_rank: int = 0,
79
+ blocksparse_local_blocks: int = 0,
80
+ blocksparse_vert_stride: int = 0,
81
+ blocksparse_block_size: int = 64,
82
+ blocksparse_head_sliding_step: int = 0,
83
) -> None:
73
- vllm_ops.paged_attention_v2(out, exp_sum, max_logits, tmp_out, query,
74
- key_cache, value_cache, num_kv_heads, scale,
75
- block_tables, seq_lens, block_size,
76
- max_seq_len, alibi_slopes, kv_cache_dtype,
77
- kv_scale)
84
+ vllm_ops.paged_attention_v2(
85
+ out, exp_sum, max_logits, tmp_out, query, key_cache, value_cache,
86
+ num_kv_heads, scale, block_tables, seq_lens, block_size, max_seq_len,
87
+ alibi_slopes, kv_cache_dtype, kv_scale, tp_rank,
88
+ blocksparse_local_blocks, blocksparse_vert_stride,
89
+ blocksparse_block_size, blocksparse_head_sliding_step)
90
91
92
# pos encoding ops
vllm/attention/backends/abstract.py
+1
@@ -111,6 +111,7 @@ class AttentionImpl(ABC, Generic[T]):
111
alibi_slopes: Optional[List[float]] = None,
112
sliding_window: Optional[int] = None,
113
kv_cache_dtype: str = "auto",
114
+ blocksparse_params: Optional[Dict[str, Any]] = None,
115
) -> None:
116
raise NotImplementedError
117
vllm/attention/backends/blocksparse_attn.py
new
+410
@@ -0,0 +1,410 @@
1
+from dataclasses import dataclass, field
2
+from typing import Any, Dict, List, Optional, Tuple, Type
3
+
4
+import torch
5
+
6
+from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
7
+ AttentionMetadata)
8
+from vllm.attention.ops.blocksparse_attention.interface import (
9
+ LocalStridedBlockSparseAttn, get_head_sliding_step)
10
+from vllm.attention.ops.paged_attn import PagedAttention
11
+from vllm.distributed import (get_tensor_model_parallel_rank,
12
+ get_tensor_model_parallel_world_size)
13
+
14
+
15
+@dataclass
16
+class BlocksparseParams:
17
+ max_seqlen: int
18
+
19
+ # Num q heads per tensor-parallel rank/partition
20
+ num_heads: int # per TP partition
21
+ # Num kv heads per tensor-parallel rank/partition
22
+ num_kv_heads: int
23
+
24
+ # block size used for blocksparse attention.
25
+ # This is the block_size used in `local_blocks`, `vert_stride`.
26
+ block_size: int
27
+
28
+ # Number of blocks for local attention, i.e., number of
29
+ # local attended tokens / `sparse_block_size`
30
+ local_blocks: int
31
+
32
+ # Attend to one block per every `vert_stride` blocks.
33
+ # Controlling the sparsity
34
+ vert_stride: int
35
+ """
36
+ If to use the same vertical stride offset for all heads,
37
+ i.e., attend to the same block of tokens on all heads.
38
+ By default, it is False, i.e., attention on the non-local
39
+ blocks depends on the `head_idx`, that is on
40
+ blocks satisfying
41
+ `(block_idx + head_idx * head_sliding_step + 1) % vert_stride == 0`
42
+ where `head_sliding_step=max(1, int(vert_stride / num_total_heads))`,
43
+ `block_idx = position_id // sparse_block_size`.
44
+ See `..ops.blocksparse_attention.utils:get_sparse_attn_mask`
45
+ for more detail.
46
+ """
47
+ homo_head: bool = False
48
+
49
+ # If within a group, the kv offsets that each q attends is the same or no.
50
+ homo_head_group: bool = False
51
+
52
+ # Decided by homo_head and homo_head group
53
+ head_sliding_step: int = field(init=False)
54
+
55
+ # range of q heads to for a TP rank
56
+ active_head_range: Tuple = field(init=False)
57
+
58
+ def __post_init__(self):
59
+ assert self.block_size > 0
60
+ assert self.local_blocks >= 0
61
+ assert self.vert_stride >= 1
62
+ assert self.num_heads % self.num_kv_heads == 0
63
+
64
+ tp_size = get_tensor_model_parallel_world_size()
65
+ tp_rank = get_tensor_model_parallel_rank()
66
+ total_heads = tp_size * self.num_heads
67
+ total_kv_heads = tp_size * self.num_kv_heads
68
+
69
+ if self.homo_head:
70
+ self.head_sliding_step = 0
71
+ elif self.homo_head_group:
72
+ head_sliding_step = get_head_sliding_step(total_kv_heads,
73
+ self.vert_stride)
74
+ # negative indicates sliding along kv heads, i.e., homo q group
75
+ self.head_sliding_step = -head_sliding_step
76
+ else:
77
+ self.head_sliding_step = get_head_sliding_step(
78
+ total_heads, self.vert_stride)
79
+
80
+ self.active_head_range = (
81
+ tp_rank * self.num_heads,
82
+ (tp_rank + 1) * self.num_heads,
83
+ )
84
+
85
+
86
+class BlocksparseFlashAttentionBackend(AttentionBackend):
87
+
88
+ @staticmethod
89
+ def get_impl_cls() -> Type["BlocksparseFlashAttentionImpl"]:
90
+ return BlocksparseFlashAttentionImpl
91
+
92
+ @staticmethod
93
+ def make_metadata(*args, **kwargs) -> "BlocksparseFlashAttentionMetadata":
94
+ return BlocksparseFlashAttentionMetadata(*args, **kwargs)
95
+
96
+ @staticmethod
97
+ def get_kv_cache_shape(
98
+ num_blocks: int,
99
+ block_size: int,
100
+ num_kv_heads: int,
101
+ head_size: int,
102
+ ) -> Tuple[int, ...]:
103
+ return PagedAttention.get_kv_cache_shape(num_blocks, block_size,
104
+ num_kv_heads, head_size)
105
+
106
+ @staticmethod
107
+ def swap_blocks(
108
+ src_kv_cache: torch.Tensor,
109
+ dst_kv_cache: torch.Tensor,
110
+ src_to_dst: Dict[int, int],
111
+ ) -> None:
112
+ PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
113
+
114
+ @staticmethod
115
+ def copy_blocks(
116
+ kv_caches: List[torch.Tensor],
117
+ src_to_dists: Dict[int, List[int]],
118
+ ) -> None:
119
+ PagedAttention.copy_blocks(kv_caches, src_to_dists)
120
+
121
+
122
+@dataclass
123
+class BlocksparseFlashAttentionMetadata(AttentionMetadata):
124
+ """A copy of Metadata for FlashAttentionBackend,
125
+ to avoid having to install flash_attn.
126
+
127
+ NOTE: Any python object stored here is not updated when it is
128
+ cuda-graph replayed. If you have values that need to be changed
129
+ dynamically, it should be stored in tensor. The tensor has to be
130
+ updated from `CUDAGraphRunner.forward` API.
131
+ """
132
+ # (batch_size,). The sequence length per sequence. Sequence length means
133
+ # the computed tokens + new tokens None if it is a decoding.
134
+ seq_lens: Optional[List[int]]
135
+ # seq_lens stored as a tensor.
136
+ seq_lens_tensor: Optional[torch.Tensor]
137
+
138
+ # NOTE(sang): Definition of context_len, query_len, and seq_len.
139
+ # |---------- N-1 iteration --------|
140
+ # |---------------- N iteration ---------------------|
141
+ # |- tokenA -|......................|-- newTokens ---|
142
+ # |---------- context_len ----------|
143
+ # |-------------------- seq_len ----------------------|
144
+ # |-- query_len ---|
145
+
146
+ # Maximum query length in the batch. None for decoding.
147
+ max_query_len: Optional[int]
148
+ # Maximum sequence length among prefill batch. 0 if there are decoding
149
+ # requests only.
150
+ max_prefill_seq_len: int
151
+ # Maximum sequence length among decode batch. 0 if there are prefill
152
+ # requests only.
153
+ max_decode_seq_len: int
154
+ # (batch_size + 1,). The cumulative subquery lengths of the sequences in
155
+ # the batch, used to index into subquery. E.g., if the subquery length
156
+ # is [4, 6], it is [0, 4, 10].
157
+ query_start_loc: Optional[torch.Tensor]
158
+ # (batch_size + 1,). The cumulative sequence lengths of the sequences in
159
+ # the batch, used to index into sequence. E.g., if the sequence length is
160
+ # [4, 6], it is [0, 4, 10].
161
+ seq_start_loc: Optional[torch.Tensor]
162
+ # (batch_size,) A tensor of context lengths (tokens that are computed
163
+ # so far).
164
+ context_lens_tensor: Optional[torch.Tensor]
165
+
166
+ # (batch_size, max_blocks_per_seq).
167
+ # Block addresses per sequence. (Seq id -> list of physical block)
168
+ # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
169
+ # in the kv cache. Each block can contain up to block_size tokens.
170
+ # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
171
+ # captured.
172
+ block_tables: Optional[torch.Tensor]
173
+
174
+ # Whether or not if cuda graph is enabled.
175
+ # Cuda-graph is currently enabled for decoding only.
176
+ # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
177
+ use_cuda_graph: bool
178
+
179
+ _cached_prefill_metadata: Optional[
180
+ "BlocksparseFlashAttentionMetadata"] = None
181
+ _cached_decode_metadata: Optional[
182
+ "BlocksparseFlashAttentionMetadata"] = None
183
+
184
+ @property
185
+ def prefill_metadata(
186
+ self) -> Optional["BlocksparseFlashAttentionMetadata"]:
187
+ if self.num_prefills == 0:
188
+ return None
189
+
190
+ if self._cached_prefill_metadata is not None:
191
+ return self._cached_prefill_metadata
192
+
193
+ assert self.seq_lens is not None
194
+ assert self.seq_lens_tensor is not None
195
+ assert self.query_start_loc is not None
196
+ assert self.context_lens_tensor is not None
197
+ assert self.block_tables is not None
198
+ assert self.seq_start_loc is not None
199
+
200
+ self._cached_prefill_metadata = BlocksparseFlashAttentionMetadata(
201
+ num_prefills=self.num_prefills,
202
+ num_prefill_tokens=self.num_prefill_tokens,
203
+ num_decode_tokens=0,
204
+ slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
205
+ seq_lens=self.seq_lens[:self.num_prefills],
206
+ seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
207
+ max_query_len=self.max_query_len,
208
+ max_prefill_seq_len=self.max_prefill_seq_len,
209
+ max_decode_seq_len=0,
210
+ query_start_loc=self.query_start_loc[:self.num_prefills + 1],
211
+ seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
212
+ context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
213
+ block_tables=self.block_tables[:self.num_prefills],
214
+ use_cuda_graph=False,
215
+ )
216
+ return self._cached_prefill_metadata
217
+
218
+ @property
219
+ def decode_metadata(self) -> Optional["BlocksparseFlashAttentionMetadata"]:
220
+ if self.num_decode_tokens == 0:
221
+ return None
222
+
223
+ if self._cached_decode_metadata is not None:
224
+ return self._cached_decode_metadata
225
+ assert self.block_tables is not None
226
+ assert self.seq_lens_tensor is not None
227
+
228
+ self._cached_decode_metadata = BlocksparseFlashAttentionMetadata(
229
+ num_prefills=0,
230
+ num_prefill_tokens=0,
231
+ num_decode_tokens=self.num_decode_tokens,
232
+ slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
233
+ seq_lens=None,
234
+ seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
235
+ max_query_len=None,
236
+ max_prefill_seq_len=0,
237
+ max_decode_seq_len=self.max_decode_seq_len,
238
+ query_start_loc=None,
239
+ seq_start_loc=None,
240
+ context_lens_tensor=None,
241
+ block_tables=self.block_tables[self.num_prefills:],
242
+ use_cuda_graph=self.use_cuda_graph,
243
+ )
244
+ return self._cached_decode_metadata
245
+
246
+
247
+class BlocksparseFlashAttentionImpl(AttentionImpl):
248
+ """
249
+ If the input tensors contain prompt tokens, the layout is as follows:
250
+ |<--------------- num_prompt_tokens -------------->|
251
+ |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|
252
+
253
+ Otherwise, the layout is as follows:
254
+ |<------------------ num_generation_tokens (M) ----------------->|
255
+ |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
256
+
257
+ Generation tokens can contain padding when cuda-graph is used.
258
+ Currently, prompt tokens don't contain any padding.
259
+
260
+ The prompts might have different lengths, while the generation tokens
261
+ always have length 1.
262
+
263
+ """
264
+
265
+ def __init__(
266
+ self,
267
+ num_heads: int,
268
+ head_size: int,
269
+ scale: float,
270
+ num_kv_heads: int,
271
+ alibi_slopes: Optional[List[float]],
272
+ sliding_window: Optional[int],
273
+ kv_cache_dtype: str,
274
+ blocksparse_params: Optional[Dict[str, Any]] = None,
275
+ ) -> None:
276
+ assert blocksparse_params is not None
277
+ assert alibi_slopes is None, ValueError(
278
+ "Alibi not support for blocksparse flash attention.")
279
+ assert sliding_window is None, ValueError(
280
+ "sliding_window is invalid for blocksparse attention.")
281
+
282
+ if "num_heads" not in blocksparse_params:
283
+ blocksparse_params["num_heads"] = num_heads
284
+ if "num_kv_heads" not in blocksparse_params:
285
+ blocksparse_params["num_kv_heads"] = num_kv_heads or num_heads
286
+ self.blocksparse_params = BlocksparseParams(**blocksparse_params)
287
+ self.kv_cache_dtype = kv_cache_dtype
288
+
289
+ self.num_heads = num_heads
290
+ self.head_size = head_size
291
+ self.scale = float(scale)
292
+ self.alibi_slopes = alibi_slopes
293
+ self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
294
+
295
+ assert self.num_heads % self.num_kv_heads == 0
296
+ self.num_queries_per_kv = self.num_heads // self.num_kv_heads
297
+
298
+ self.local_blocks = self.blocksparse_params.local_blocks
299
+ self.vert_stride = self.blocksparse_params.vert_stride
300
+ self.sparse_block_size = self.blocksparse_params.block_size
301
+ self.head_sliding_step = self.blocksparse_params.head_sliding_step
302
+
303
+ suppored_head_sizes = PagedAttention.get_supported_head_sizes()
304
+ if head_size not in suppored_head_sizes:
305
+ raise ValueError(
306
+ f"Head size {head_size} is not supported by PagedAttention. "
307
+ f"Supported head sizes are: {suppored_head_sizes}.")
308
+
309
+ self.tp_size = get_tensor_model_parallel_world_size()
310
+ self.tp_rank = get_tensor_model_parallel_rank()
311
+
312
+ total_num_heads = num_heads * self.tp_size
313
+ self.bs_attn = LocalStridedBlockSparseAttn(
314
+ total_num_heads,
315
+ self.blocksparse_params.max_seqlen,
316
+ self.blocksparse_params.local_blocks,
317
+ self.blocksparse_params.vert_stride,
318
+ self.blocksparse_params.block_size,
319
+ homo_head=self.blocksparse_params.homo_head,
320
+ active_head_range=self.blocksparse_params.active_head_range,
321
+ )
322
+
323
+ def forward(
324
+ self,
325
+ query: torch.Tensor,
326
+ key: torch.Tensor,
327
+ value: torch.Tensor,
328
+ kv_cache: torch.Tensor,
329
+ attn_metadata: BlocksparseFlashAttentionMetadata,
330
+ kv_scale: float = 1.0,
331
+ ) -> torch.Tensor:
332
+ """Forward pass with FlashAttention and PagedAttention.
333
+
334
+ Args:
335
+ query: shape = [num_tokens, num_heads * head_size]
336
+ key: shape = [num_tokens, num_kv_heads * head_size]
337
+ value: shape = [num_tokens, num_kv_heads * head_size]
338
+ kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
339
+ attn_metadata: Metadata for attention.
340
+ Returns:
341
+ shape = [num_tokens, num_heads * head_size]
342
+ """
343
+ num_tokens, hidden_size = query.shape
344
+ # Reshape the query, key, and value tensors.
345
+ query = query.view(-1, self.num_heads, self.head_size)
346
+ key = key.view(-1, self.num_kv_heads, self.head_size)
347
+ value = value.view(-1, self.num_kv_heads, self.head_size)
348
+
349
+ if kv_cache is not None:
350
+ key_cache, value_cache = PagedAttention.split_kv_cache(
351
+ kv_cache, self.num_kv_heads, self.head_size)
352
+
353
+ # Reshape the input keys and values and store them in the cache.
354
+ # If kv_cache is not provided, the new key and value tensors are
355
+ # not cached. This happens during the initial memory profiling run.
356
+
357
+ PagedAttention.write_to_paged_cache(
358
+ key,
359
+ value,
360
+ key_cache,
361
+ value_cache,
362
+ attn_metadata.slot_mapping,
363
+ self.kv_cache_dtype,
364
+ kv_scale,
365
+ )
366
+
367
+ if prefill_meta := attn_metadata.prefill_metadata:
368
+
369
+ # Prompt run.
370
+ # normal attention
371
+ # When block_tables are not filled, it means q and k are the
372
+ # prompt, and they have the same length.
373
+
374
+ assert kv_cache is None \
375
+ or prefill_meta.block_tables is None \
376
+ or prefill_meta.block_tables.numel() == 0, \
377
+ "Does not support prefix-enabled attention."
378
+
379
+ output = self.bs_attn(
380
+ q=query,
381
+ k=key,
382
+ v=value,
383
+ cu_seqlens_q=prefill_meta.seq_start_loc,
384
+ cu_seqlens_k=prefill_meta.seq_start_loc,
385
+ sm_scale=self.scale,
386
+ )
387
+
388
+ if decode_meta := attn_metadata.decode_metadata:
389
+ # Decoding run.
390
+ output = PagedAttention.forward_decode(
391
+ query,
392
+ key_cache,
393
+ value_cache,
394
+ decode_meta.block_tables,
395
+ decode_meta.seq_lens_tensor,
396
+ self.blocksparse_params.max_seqlen,
397
+ self.kv_cache_dtype,
398
+ self.num_kv_heads,
399
+ self.scale,
400
+ self.alibi_slopes,
401
+ kv_scale,
402
+ tp_rank=self.tp_rank,
403
+ blocksparse_local_blocks=self.local_blocks,
404
+ blocksparse_vert_stride=self.vert_stride,
405
+ blocksparse_block_size=self.sparse_block_size,
406
+ blocksparse_head_sliding_step=self.head_sliding_step,
407
+ )
408
+
409
+ # Reshape the output tensor.
410
+ return output.view(num_tokens, hidden_size)
vllm/attention/backends/flash_attn.py
+4
-1
@@ -1,6 +1,6 @@
1
"""Attention layer with FlashAttention."""
2
from dataclasses import dataclass
3
-from typing import List, Optional, Tuple, Type
3
+from typing import Any, Dict, List, Optional, Tuple, Type
4
5
import torch
6
from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
@@ -219,7 +219,10 @@ class FlashAttentionImpl(AttentionImpl):
219
alibi_slopes: Optional[List[float]],
220
sliding_window: Optional[int],
221
kv_cache_dtype: str,
222
+ blocksparse_params: Optional[Dict[str, Any]] = None,
223
) -> None:
224
+ assert blocksparse_params is None, ValueError(
225
+ "FlashAttention does not support block-sparse attention.")
226
self.num_heads = num_heads
227
self.head_size = head_size
228
self.scale = float(scale)
vllm/attention/backends/rocm_flash_attn.py
+4
-1
@@ -1,6 +1,6 @@
1
"""Attention layer ROCm GPUs."""
2
from dataclasses import dataclass
3
-from typing import List, Optional, Tuple, Type
3
+from typing import Any, Dict, List, Optional, Tuple, Type
4
5
import torch
6
@@ -201,7 +201,10 @@ class ROCmFlashAttentionImpl(AttentionImpl):
201
alibi_slopes: Optional[List[float]],
202
sliding_window: Optional[int],
203
kv_cache_dtype: str,
204
+ blocksparse_params: Optional[Dict[str, Any]] = None,
205
) -> None:
206
+ assert blocksparse_params is None, ValueError(
207
+ "ROCFlashAttention does not support blocksparse attention.")
208
self.num_heads = num_heads
209
self.head_size = head_size
210
self.scale = float(scale)
vllm/attention/backends/torch_sdpa.py
+4
-1
@@ -1,7 +1,7 @@
1
""" Attention layer with torch scaled_dot_product_attention
2
and PagedAttention."""
3
from dataclasses import dataclass
4
-from typing import List, Optional, Tuple, Type
4
+from typing import Any, Dict, List, Optional, Tuple, Type
5
6
import torch
7
from torch.nn.functional import scaled_dot_product_attention
@@ -100,7 +100,10 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
100
alibi_slopes: Optional[List[float]],
101
sliding_window: Optional[int],
102
kv_cache_dtype: str,
103
+ blocksparse_params: Optional[Dict[str, Any]] = None,
104
) -> None:
105
+ assert blocksparse_params is None, ValueError(
106
+ "Torch SPDA does not support block-sparse attention.")
107
self.num_heads = num_heads
108
self.head_size = head_size
109
self.scale = float(scale)
vllm/attention/backends/xformers.py
+4
-1
@@ -1,6 +1,6 @@
1
"""Attention layer with xFormers and PagedAttention."""
2
from dataclasses import dataclass
3
-from typing import Dict, List, Optional, Tuple, Type
3
+from typing import Any, Dict, List, Optional, Tuple, Type
4
5
import torch
6
from xformers import ops as xops
@@ -212,7 +212,10 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
212
alibi_slopes: Optional[List[float]],
213
sliding_window: Optional[int],
214
kv_cache_dtype: str,
215
+ blocksparse_params: Optional[Dict[str, Any]] = None,
216
) -> None:
217
+ assert blocksparse_params is None, ValueError(
218
+ "XFormer does not support block-sparse attention.")
219
self.num_heads = num_heads
220
self.head_size = head_size
221
self.scale = float(scale)
vllm/attention/layer.py
+7
-3
@@ -1,5 +1,5 @@
1
"""Attention layer."""
2
-from typing import List, Optional
2
+from typing import Any, Dict, List, Optional
3
4
import torch
5
import torch.nn as nn
@@ -33,6 +33,7 @@ class Attention(nn.Module):
33
sliding_window: Optional[int] = None,
34
cache_config: Optional[CacheConfig] = None,
35
quant_config: Optional[QuantizationConfig] = None,
36
+ blocksparse_params: Optional[Dict[str, Any]] = None,
37
) -> None:
38
super().__init__()
39
if cache_config is not None:
@@ -69,10 +70,12 @@ class Attention(nn.Module):
70
dtype = torch.get_default_dtype()
71
attn_backend = get_attn_backend(num_heads, head_size, num_kv_heads,
72
sliding_window, dtype, kv_cache_dtype,
72
- block_size)
73
+ block_size, blocksparse_params
74
+ is not None)
75
impl_cls = attn_backend.get_impl_cls()
76
self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
75
- alibi_slopes, sliding_window, kv_cache_dtype)
77
+ alibi_slopes, sliding_window, kv_cache_dtype,
78
+ blocksparse_params)
79
80
def forward(
81
self,
@@ -90,4 +93,5 @@ class Attention(nn.Module):
93
s += f", num_heads={self.impl.num_heads}" # type: ignore
94
s += f", num_kv_heads={self.impl.num_kv_heads}" # type: ignore
95
s += f", scale={self.impl.scale}" # type: ignore
96
+ s += f", backend={self.impl.__class__.__name__}"
97
return s
vllm/attention/ops/blocksparse_attention/__init__.py
vllm/attention/ops/blocksparse_attention/blocksparse_attention_kernel.py
new
+423
@@ -0,0 +1,423 @@
1
+import torch
2
+import triton
3
+import triton.language as tl
4
+
5
+
6
+def blocksparse_flash_attn_varlen_fwd(
7
+ q,
8
+ k,
9
+ v, # (#tokens, n_heads, head_size)
10
+ cu_seqlens_k,
11
+ cu_seqlens_q,
12
+ sm_scale,
13
+ sparse_layout,
14
+ *,
15
+ block_size=64,
16
+ q_block_size=None,
17
+ max_seqlen=None):
18
+ # split q to blocks
19
+
20
+ assert isinstance(sparse_layout, (list, tuple))
21
+
22
+ _, n_heads, head_size = q.shape
23
+ batch_size = cu_seqlens_k.size(0) - 1
24
+ q_block_size = q_block_size or block_size
25
+
26
+ assert q.dim() == k.dim() == v.dim() == 3
27
+ assert q.size(1) % k.size(1) == 0
28
+ assert q.size(2) == k.size(2)
29
+ # TODO(linxihui): allow k, v to have different head_size
30
+ assert k.shape == v.shape
31
+ assert cu_seqlens_k.dim() == 1
32
+
33
+ q_k_ratio = q.size(1) // k.size(1)
34
+
35
+ if cu_seqlens_q is None:
36
+ if q.size(0) == batch_size: # decoding only
37
+ cu_seqlens_q = torch.arange(
38
+ 0,
39
+ batch_size + 1,
40
+ dtype=cu_seqlens_k.dtype,
41
+ device=cu_seqlens_k.device,
42
+ )
43
+ elif q.size(0) == k.size(0):
44
+ cu_seqlens_q = cu_seqlens_k
45
+ else:
46
+ raise ValueError("cu_seqlens_q must be specified\
47
+ if it mix of prefilling and decoding.")
48
+ else:
49
+ assert cu_seqlens_k.size(0) == cu_seqlens_q.size(0)
50
+
51
+ # switch to use cpu to avoid too many kernel launches when iterated over
52
+ q_lens = (cu_seqlens_q[1:] - cu_seqlens_q[:-1]).cpu()
53
+ k_lens = (cu_seqlens_k[1:] - cu_seqlens_k[:-1]).cpu()
54
+
55
+ assert torch.logical_or(q_lens == 1, k_lens == q_lens).all(), (
56
+ "length of q should either be 1 (decoding) or same as k (prefilling).")
57
+
58
+ if max_seqlen:
59
+ assert k_lens.max() <= max_seqlen
60
+
61
+ n_blocks = (q_lens + q_block_size - 1) // q_block_size
62
+
63
+ q_batch_ids = torch.tensor(
64
+ [i for i, n in enumerate(n_blocks) for _ in range(n)],
65
+ dtype=cu_seqlens_q.dtype,
66
+ device=cu_seqlens_q.device,
67
+ )
68
+ q_start_sids = torch.tensor(
69
+ [i * q_block_size for n in n_blocks for i in range(n)],
70
+ dtype=cu_seqlens_q.dtype,
71
+ device=cu_seqlens_q.device,
72
+ )
73
+
74
+ out = q.new_empty(q.shape)
75
+ cu_seqlens_q = cu_seqlens_q.contiguous()
76
+ cu_seqlens_k = cu_seqlens_k.contiguous()
77
+
78
+ layout_crow_indices, layout_col_indices = sparse_layout
79
+ block_d = triton.next_power_of_2(head_size)
80
+
81
+ decoding_only = (q_lens == 1).all().item()
82
+ grid = (len(q_start_sids), n_heads, 1)
83
+
84
+ _fwd_kernel_batch_inference[grid](
85
+ q,
86
+ k,
87
+ v,
88
+ out,
89
+ sm_scale,
90
+ cu_seqlens_q[:-1],
91
+ cu_seqlens_q[1:],
92
+ cu_seqlens_k[:-1],
93
+ cu_seqlens_k[1:],
94
+ q_batch_ids,
95
+ q_start_sids,
96
+ 0,
97
+ *q.stride(),
98
+ 0,
99
+ *k.stride(),
100
+ 0,
101
+ *v.stride(),
102
+ 0,
103
+ *out.stride(),
104
+ layout_crow_indices,
105
+ layout_col_indices,
106
+ *layout_crow_indices.stride(),
107
+ *layout_col_indices.stride(),
108
+ q_k_ratio,
109
+ HAS_BATCH_DIM=False,
110
+ D_HEAD=head_size,
111
+ BLOCK_M=q_block_size,
112
+ BLOCK_N=block_size,
113
+ BLOCK_D=block_d,
114
+ BLOCK_M_LOADING=(16 if decoding_only else
115
+ q_block_size), # smaller for decoding
116
+ EVEN_D=block_d == head_size,
117
+ num_warps=1 if decoding_only else 4,
118
+ num_stages=3)
119
+
120
+ return out
121
+
122
+
123
+@triton.jit
124
+def _fwd_kernel_inner(
125
+ acc,
126
+ l_i,
127
+ m_i,
128
+ q,
129
+ Q,
130
+ k_block_col_idx,
131
+ layout_col_ptr,
132
+ layout_col_stride_h,
133
+ layout_col_stride_m,
134
+ k_ptrs,
135
+ v_ptrs,
136
+ off_h,
137
+ offs_m,
138
+ offs_n,
139
+ offs_d,
140
+ stride_kt,
141
+ stride_vt,
142
+ sm_scale,
143
+ k_seqlen,
144
+ past_len,
145
+ LAST_K_BLOCK: tl.constexpr,
146
+ BLOCK_M_LOADING: tl.constexpr,
147
+ BLOCK_N: tl.constexpr,
148
+ D_HEAD: tl.constexpr,
149
+ EVEN_D: tl.constexpr,
150
+ M_LT_N: tl.constexpr,
151
+):
152
+ k_block_id = tl.load(layout_col_ptr + off_h * layout_col_stride_h +
153
+ k_block_col_idx * layout_col_stride_m).to(tl.int32)
154
+ start_n = k_block_id * BLOCK_N
155
+ if LAST_K_BLOCK:
156
+ if EVEN_D:
157
+ k = tl.load(
158
+ k_ptrs + start_n * stride_kt,
159
+ mask=offs_n[None, :] + start_n < k_seqlen,
160
+ )
161
+ else:
162
+ k = tl.load(
163
+ k_ptrs + start_n * stride_kt,
164
+ mask=(offs_n[None, :] + start_n < k_seqlen) &
165
+ (offs_d[:, None] < D_HEAD),
166
+ )
167
+ else:
168
+ if EVEN_D:
169
+ k = tl.load(k_ptrs + start_n * stride_kt)
170
+ else:
171
+ k = tl.load(k_ptrs + start_n * stride_kt,
172
+ mask=offs_d[:, None] < D_HEAD)
173
+
174
+ qk = tl.zeros([BLOCK_M_LOADING, BLOCK_N], dtype=tl.float32)
175
+ qk += tl.dot(q, k)
176
+ qk *= sm_scale
177
+
178
+ # the following is needed only when LAST_K_BLOCK or BLOCK_M < BLOCK_N
179
+ if LAST_K_BLOCK | M_LT_N:
180
+ qk += tl.where(
181
+ offs_m[:, None] + past_len >= (start_n + offs_n[None, :]),
182
+ 0,
183
+ float("-inf"),
184
+ )
185
+
186
+ # flash-attn2
187
+ m_ij = tl.maximum(m_i, tl.max(qk, 1))
188
+ p = tl.math.exp2(qk - m_ij[:, None])
189
+ l_ij = tl.sum(p, 1)
190
+ alpha = tl.math.exp2(m_i - m_ij)
191
+ acc = acc * alpha[:, None]
192
+ # update m_i
193
+ m_i = m_ij
194
+ l_i = l_i * alpha + l_ij
195
+
196
+ p = p.to(Q.dtype.element_ty)
197
+ # update acc
198
+ if LAST_K_BLOCK:
199
+ if EVEN_D:
200
+ v = tl.load(
201
+ v_ptrs + start_n * stride_vt,
202
+ mask=offs_n[:, None] + start_n < k_seqlen,
203
+ )
204
+ else:
205
+ v = tl.load(
206
+ v_ptrs + start_n * stride_vt,
207
+ mask=(offs_n[:, None] + start_n < k_seqlen) &
208
+ (offs_d[None, :] < D_HEAD),
209
+ )
210
+ else:
211
+ if EVEN_D:
212
+ v = tl.load(v_ptrs + start_n * stride_vt)
213
+ else:
214
+ v = tl.load(v_ptrs + start_n * stride_vt,
215
+ mask=offs_d[None, :] < D_HEAD)
216
+
217
+ acc += tl.dot(p, v)
218
+
219
+ return acc, l_i, m_i
220
+
221
+
222
+@triton.heuristics({
223
+ "M_LT_N":
224
+ lambda kwargs: kwargs["BLOCK_M"] < kwargs["BLOCK_N"],
225
+})
226
+@triton.jit
227
+def _fwd_kernel_batch_inference(
228
+ Q,
229
+ K,
230
+ V,
231
+ Out,
232
+ sm_scale,
233
+ q_batch_starts,
234
+ q_batch_ends,
235
+ k_batch_starts,
236
+ k_batch_ends,
237
+ q_batch_ids,
238
+ q_start_sids,
239
+ stride_qb,
240
+ stride_qt,
241
+ stride_qh,
242
+ stride_qd,
243
+ stride_kb,
244
+ stride_kt,
245
+ stride_kh,
246
+ stride_kd,
247
+ stride_vb,
248
+ stride_vt,
249
+ stride_vh,
250
+ stride_vd,
251
+ stride_ob,
252
+ stride_ot,
253
+ stride_oh,
254
+ stride_od,
255
+ layout_crow_ptr,
256
+ layout_col_ptr,
257
+ layout_crow_stride_h,
258
+ layout_crow_stride_m,
259
+ layout_col_stride_h,
260
+ layout_col_stride_m,
261
+ q_k_ratio,
262
+ HAS_BATCH_DIM: tl.constexpr,
263
+ D_HEAD: tl.constexpr,
264
+ BLOCK_M: tl.constexpr,
265
+ BLOCK_N: tl.constexpr,
266
+ BLOCK_D: tl.constexpr,
267
+ BLOCK_M_LOADING: tl.constexpr,
268
+ EVEN_D: tl.constexpr,
269
+ M_LT_N: tl.constexpr,
270
+):
271
+ """
272
+ NOTATION:
273
+ pid: position id
274
+ sid: storage id
275
+ sbid: storage block id
276
+ pbid: position block id
277
+ offs_m, offs_n: storage offsets of m-dim(q, row) and n-dim(k, col)
278
+
279
+ TODO(linxihui):
280
+ Optimize grouped-attn
281
+ """
282
+ off_zm = tl.program_id(0)
283
+ off_h = tl.program_id(1)
284
+
285
+ off_h_for_kv = off_h // q_k_ratio
286
+
287
+ if HAS_BATCH_DIM:
288
+ off_z = tl.program_id(2)
289
+ Q += off_z * stride_qb
290
+ K += off_z * stride_kb
291
+ V += off_z * stride_vb
292
+ Out += off_z * stride_ob
293
+ start_m = off_zm
294
+ q_start_sid = start_m * BLOCK_M # always 0 for decoding
295
+ else:
296
+ off_z = tl.load(q_batch_ids + off_zm).to(tl.int32) # [0, 0, 0, 1]
297
+ q_start_sid = tl.load(q_start_sids + off_zm)
298
+ start_m = q_start_sid // BLOCK_M # q_sbid
299
+
300
+ offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M_LOADING)
301
+ offs_n = tl.arange(0, BLOCK_N)
302
+ offs_d = tl.arange(0, BLOCK_D)
303
+
304
+ q_cu_start = tl.load(q_batch_starts + off_z).to(tl.int32)
305
+ q_seqlen = tl.load(q_batch_ends + off_z).to(tl.int32) - q_cu_start
306
+ k_cu_start = tl.load(k_batch_starts + off_z).to(tl.int32)
307
+ k_seqlen = tl.load(k_batch_ends + off_z).to(tl.int32) - k_cu_start
308
+ past_len = k_seqlen - q_seqlen
309
+
310
+ Q += q_cu_start * stride_qt + off_h * stride_qh
311
+ K += k_cu_start * stride_kt + off_h_for_kv * stride_kh
312
+ V += k_cu_start * stride_vt + off_h_for_kv * stride_vh
313
+ Out += q_cu_start * stride_ot + off_h * stride_oh
314
+
315
+ q_pbid = (past_len + q_start_sid) // BLOCK_M
316
+
317
+ if EVEN_D:
318
+ q = tl.load(
319
+ Q + offs_m[:, None] * stride_qt + offs_d[None, :] * stride_qd,
320
+ mask=offs_m[:, None] < q_seqlen,
321
+ )
322
+ else:
323
+ q = tl.load(
324
+ Q + offs_m[:, None] * stride_qt + offs_d[None, :] * stride_qd,
325
+ mask=(offs_m[:, None] < q_seqlen) & (offs_d[None, :] < D_HEAD),
326
+ other=0,
327
+ )
328
+
329
+ sparse_crow_ptr = (layout_crow_ptr + off_h * layout_crow_stride_h +
330
+ q_pbid * layout_crow_stride_m)
331
+
332
+ # TODO(linxihui): load at once, with any Triton version
333
+ # that supports `tl.split`, e.g., Triton 3.0
334
+ k_block_start = tl.load(sparse_crow_ptr).to(tl.int32)
335
+ k_block_end = tl.load(sparse_crow_ptr + 1).to(tl.int32)
336
+
337
+ m_i = tl.zeros([BLOCK_M_LOADING], dtype=tl.float32) - float("inf")
338
+ l_i = tl.zeros([BLOCK_M_LOADING], dtype=tl.float32)
339
+ acc = tl.zeros([BLOCK_M_LOADING, BLOCK_D], dtype=tl.float32)
340
+
341
+ k_ptrs = K + offs_n[None, :] * stride_kt + offs_d[:, None] * stride_kd
342
+ v_ptrs = V + offs_n[:, None] * stride_vt + offs_d[None, :] * stride_vd
343
+
344
+ sm_scale *= (
345
+ 1.44269504 # 1/log2 as we use base2 for exponential and logarithm
346
+ )
347
+
348
+ for k_block_col_idx in range(k_block_start, k_block_end - 1):
349
+ acc, l_i, m_i = _fwd_kernel_inner(
350
+ acc,
351
+ l_i,
352
+ m_i,
353
+ q,
354
+ Q,
355
+ k_block_col_idx,
356
+ layout_col_ptr,
357
+ layout_col_stride_h,
358
+ layout_col_stride_m,
359
+ k_ptrs,
360
+ v_ptrs,
361
+ off_h,
362
+ offs_m,
363
+ offs_n,
364
+ offs_d,
365
+ stride_kt,
366
+ stride_vt,
367
+ sm_scale,
368
+ k_seqlen,
369
+ past_len,
370
+ False,
371
+ BLOCK_M_LOADING,
372
+ BLOCK_N,
373
+ D_HEAD,
374
+ EVEN_D,
375
+ M_LT_N,
376
+ )
377
+
378
+ acc, l_i, m_i = _fwd_kernel_inner(
379
+ acc,
380
+ l_i,
381
+ m_i,
382
+ q,
383
+ Q,
384
+ k_block_end - 1,
385
+ layout_col_ptr,
386
+ layout_col_stride_h,
387
+ layout_col_stride_m,
388
+ k_ptrs,
389
+ v_ptrs,
390
+ off_h,
391
+ offs_m,
392
+ offs_n,
393
+ offs_d,
394
+ stride_kt,
395
+ stride_vt,
396
+ sm_scale,
397
+ k_seqlen,
398
+ past_len,
399
+ True,
400
+ BLOCK_M_LOADING,
401
+ BLOCK_N,
402
+ D_HEAD,
403
+ EVEN_D,
404
+ M_LT_N,
405
+ )
406
+
407
+ # flash-attn 2
408
+ m_i += tl.math.log2(l_i)
409
+ acc = acc / l_i[:, None]
410
+
411
+ # write output
412
+ if EVEN_D:
413
+ tl.store(
414
+ Out + offs_m[:, None] * stride_ot + offs_d[None, :] * stride_od,
415
+ acc,
416
+ mask=offs_m[:, None] < q_seqlen,
417
+ )
418
+ else:
419
+ tl.store(
420
+ Out + offs_m[:, None] * stride_ot + offs_d[None, :] * stride_od,
421
+ acc,
422
+ mask=(offs_m[:, None] < q_seqlen) & (offs_d[None, :] < D_HEAD),
423
+ )
vllm/attention/ops/blocksparse_attention/interface.py
new
+238
@@ -0,0 +1,238 @@
1
+import math
2
+
3
+import torch
4
+
5
+from vllm.utils import is_cpu, is_hip
6
+
7
+from .utils import (dense_to_crow_col, get_head_sliding_step,
8
+ get_sparse_attn_mask)
9
+
10
+IS_COMPUTE_8_OR_ABOVE = (torch.cuda.is_available()
11
+ and torch.cuda.get_device_capability()[0] >= 8)
12
+
13
+if IS_COMPUTE_8_OR_ABOVE:
14
+ from .blocksparse_attention_kernel import blocksparse_flash_attn_varlen_fwd
15
+
16
+
17
+class LocalStridedBlockSparseAttn(torch.nn.Module):
18
+
19
+ def __init__(
20
+ self,
21
+ n_heads,
22
+ max_seqlen,
23
+ local_blocks,
24
+ vert_stride,
25
+ block_size,
26
+ device=None,
27
+ dtype=None,
28
+ homo_head=False,
29
+ active_head_range=None,
30
+ q_block_size=None,
31
+ use_spda=None,
32
+ ):
33
+ super().__init__()
34
+ if use_spda is None:
35
+ use_spda = is_hip() or is_cpu() or not \
36
+ IS_COMPUTE_8_OR_ABOVE
37
+ device = device or (torch.cuda.current_device()
38
+ if torch.cuda.is_available() else "cpu")
39
+ device = torch.device(device)
40
+ # NOTE: vllm CPU backend support BF16 instead of FP16.
41
+ dtype = dtype or (torch.bfloat16 if IS_COMPUTE_8_OR_ABOVE
42
+ or device.type == "cpu" else torch.half)
43
+
44
+ self.n_heads = n_heads
45
+ self.max_seqlen = max_seqlen
46
+ self.local_blocks = local_blocks
47
+ self.vert_stride = vert_stride
48
+ self.use_spda = use_spda
49
+ self.dtype = dtype
50
+ self.device = device
51
+ self.block_size = block_size
52
+ self.q_block_size = q_block_size
53
+ self.homo_head = homo_head
54
+ self.active_head_range = active_head_range
55
+ self.head_sliding_step = get_head_sliding_step(n_heads, vert_stride,
56
+ homo_head)
57
+
58
+ sparse_layout, sparse_pattern, self.dense_attn_mask = (
59
+ self.get_attn_pattern(dtype, device))
60
+
61
+ if q_block_size is not None and q_block_size != block_size:
62
+ if q_block_size > block_size:
63
+ assert q_block_size % block_size == 0
64
+ blocks_to_merge = q_block_size // block_size
65
+ shape = sparse_pattern.shape
66
+ sparse_pattern = sparse_pattern.view(shape[0], -1,
67
+ blocks_to_merge,
68
+ shape[-1])
69
+ sparse_pattern = sparse_pattern.sum(2)
70
+ sparse_layout = dense_to_crow_col(sparse_pattern)
71
+ else:
72
+ raise ValueError(
73
+ "Does not support smaller q_block_size. It will be slower."
74
+ )
75
+
76
+ self.sparse_layout = sparse_layout
77
+
78
+ def get_attn_pattern(self, dtype, device):
79
+ sparse_layout, sparse_pattern, dense_attn_mask = get_sparse_attn_mask(
80
+ self.n_heads,
81
+ self.max_seqlen,
82
+ self.max_seqlen,
83
+ dtype,
84
+ device,
85
+ block_size=self.block_size,
86
+ local_blocks=self.local_blocks,
87
+ vert_stride=self.vert_stride,
88
+ homo_head=self.homo_head,
89
+ return_dense=self.use_spda,
90
+ dense_mask_type="bias",
91
+ )
92
+ if (not self.homo_head) and (self.active_head_range is not None):
93
+ assert isinstance(self.active_head_range, tuple)
94
+ assert (len(self.active_head_range) == 2)
95
+ h_start, h_end = self.active_head_range
96
+ sparse_layout = tuple(x[h_start:h_end] for x in sparse_layout)
97
+ if self.use_spda:
98
+ dense_attn_mask = dense_attn_mask[h_start:h_end]
99
+ return sparse_layout, sparse_pattern, dense_attn_mask
100
+
101
+ def varlen_attn(self,
102
+ q,
103
+ k,
104
+ v,
105
+ cu_seqlens_k,
106
+ cu_seqlens_q=None,
107
+ sm_scale=None):
108
+ """
109
+ q, k, v: shape = (num_tokens, num_heads_q/kv, head_size).
110
+ Support grouped attention, with `q[:, i*r:(i*r + r)]`
111
+ is correspondent to `k[:, i]`, where `r` is the q/k ratio.
112
+ cu_seqlens_k: shape=(batch_size + 1,),
113
+ indicating segment of samples,
114
+ e.g., `k[cu_seqlen[i]:cu_seqlne[i+1]]` is q of sample i
115
+ cu_seqlens_q: shape=(batch_size + 1, ).
116
+ Default None: same as cu_seqlens_k for prefilling or
117
+ [0, 1, .., batch_size] for decoding.
118
+ The only case you need to specify is when q is a mix of
119
+ prefilling and decoding.
120
+ sm_scale: softmax scale, default to 1/sqrt(head_size).
121
+
122
+ return: tensor of shape as q.
123
+ """
124
+ assert (
125
+ IS_COMPUTE_8_OR_ABOVE
126
+ ), "Requires compute capability of 8 or above (Ampere or newer) to use \
127
+ Triton kernel."
128
+
129
+ sm_scale = sm_scale or 1.0 / math.sqrt(q.size(-1))
130
+
131
+ return blocksparse_flash_attn_varlen_fwd(
132
+ q,
133
+ k,
134
+ v,
135
+ cu_seqlens_k,
136
+ cu_seqlens_q,
137
+ sm_scale,
138
+ self.sparse_layout,
139
+ block_size=self.block_size,
140
+ q_block_size=self.q_block_size,
141
+ max_seqlen=self.max_seqlen,
142
+ )
143
+
144
+ @staticmethod
145
+ def transpose_and_pad(x, cu_seqlens, maxlen, head_repeats=1):
146
+ """
147
+ :param x: (total_tokens, n_heads, head_size)
148
+ :return: (batch, n_heads, length, head_size)
149
+ """
150
+ x_padded = x.new_empty(
151
+ len(cu_seqlens) - 1, x.size(1), head_repeats, maxlen, x.size(2))
152
+ cu_seqlens = cu_seqlens.cpu()
153
+ for i, (s, e) in enumerate(zip(cu_seqlens[:-1], cu_seqlens[1:])):
154
+ x_padded[i, :, :, :e - s].copy_(x[s:e].transpose(0,
155
+ 1).unsqueeze(1))
156
+ return x_padded.flatten(1, 2)
157
+
158
+ @staticmethod
159
+ def transpose_and_unpad(x_padded, cu_seqlens):
160
+ """
161
+ :param x_padded: (batch, n_heads, length, head_size)
162
+ :return: (total_tokens, n_heads, head_size)
163
+ """
164
+ cu_seqlens = cu_seqlens.cpu()
165
+ total_n_tokens = cu_seqlens[-1]
166
+ x = x_padded.new_empty(total_n_tokens, x_padded.size(1),
167
+ x_padded.size(3))
168
+ for i, (s, e) in enumerate(zip(cu_seqlens[:-1], cu_seqlens[1:])):
169
+ x[s:e].copy_(x_padded[i, :, :e - s].transpose(0, 1))
170
+ return x
171
+
172
+ def spda(self, q, k, v, cu_seqlens_k, cu_seqlens_q=None, sm_scale=None):
173
+ """For CPU, V100 or other older GPUs.
174
+ NOTE: torch SPDA supports nested tensor,
175
+ but seems extremely slow. Choose to pad instead.
176
+ """
177
+ assert (cu_seqlens_q is None or
178
+ (cu_seqlens_q
179
+ == cu_seqlens_k).all()), "Can only handle prompt with SPDA."
180
+ assert q.size(0) == k.size(0), "can only handle prompt with SPDA."
181
+
182
+ assert q.size(1) % k.size(1) == 0
183
+ q_k_ratio = q.size(1) // k.size(1)
184
+ sm_scale = sm_scale or 1.0 / math.sqrt(q.size(-1))
185
+ cu_seqlens = cu_seqlens_k.cpu()
186
+ maxlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max()
187
+
188
+ if (self.dense_attn_mask.dtype != q.dtype
189
+ or self.dense_attn_mask.device != q.device):
190
+ _, _, self.dense_attn_mask = self.get_attn_pattern(
191
+ q.dtype, q.device)
192
+ attn_mask = self.dense_attn_mask[None, :, :maxlen, :maxlen]
193
+
194
+ q2 = self.transpose_and_pad(q, cu_seqlens, maxlen, 1)
195
+ k2, v2 = [
196
+ self.transpose_and_pad(x, cu_seqlens, maxlen, q_k_ratio)
197
+ for x in [k, v]
198
+ ]
199
+ spda_output = torch.nn.functional.scaled_dot_product_attention(
200
+ q2, k2, v2, attn_mask=attn_mask, scale=sm_scale)
201
+ return self.transpose_and_unpad(spda_output, cu_seqlens)
202
+
203
+ def forward(self, q, k, v, cu_seqlens_k, cu_seqlens_q=None, sm_scale=None):
204
+ """Dispatch to `varlen_attn` (Ampere or newer) or
205
+ `self.spda`(cpu, Volta, Turing or older)based on
206
+ the type of device used and cuda compute capability.
207
+
208
+ q, k, v: shape = (num_tokens, num_heads_q/kv, head_size).
209
+ Support grouped attention, with `q[:, i*r:(i*r + r)]`
210
+ is correspondent to `k[:, i]`, where `r` is the q/k ratio.
211
+ cu_seqlens_k: shape=(batch_size + 1,), indicating segment of samples,
212
+ e.g., `k[cu_seqlen[i]:cu_seqlne[i+1]]` is q of sample i
213
+ cu_seqlens_q: shape=(batch_size + 1, ).
214
+ Default None: same as cu_seqlens_k for prefilling or
215
+ [0, 1, .., batch_size] for decoding.
216
+ The only case you need to specify
217
+ is when q is a mix of prefilling
218
+ and decoding.
219
+ sm_scale: softmax scale, default to 1/sqrt(head_size).
220
+
221
+ return: tensor of shape as q.
222
+ """
223
+ assert k.dim() == 3
224
+ if self.use_spda:
225
+ return self.spda(
226
+ q,
227
+ k,
228
+ v,
229
+ cu_seqlens_k,
230
+ cu_seqlens_q=cu_seqlens_q,
231
+ sm_scale=sm_scale,
232
+ )
233
+ return self.varlen_attn(q,
234
+ k,
235
+ v,
236
+ cu_seqlens_k,
237
+ cu_seqlens_q=cu_seqlens_q,
238
+ sm_scale=sm_scale)
\ No newline at end of file
vllm/attention/ops/blocksparse_attention/utils.py
new
+216
@@ -0,0 +1,216 @@
1
+# Helper functions for 3D sparse pattern
2
+# These function are not optimized and very inefficient.
3
+# Avoid calling them too frequent or use a cache mechanism.
4
+
5
+from functools import lru_cache
6
+
7
+import torch
8
+import triton
9
+from scipy import sparse
10
+
11
+
12
+def dense_to_crow_col(x: torch.Tensor):
13
+ """Turning a 2D/3D torch tensor (x) to CSR rows/cols indexing.
14
+ NOTE: col_indices padded -1
15
+ """
16
+ device = x.device
17
+ pad = -1
18
+ dim = x.dim()
19
+ assert x.dim() in (2, 3)
20
+ if x.dim() == 2:
21
+ x = x[None]
22
+ x = [sparse.csr_matrix(xi.bool().cpu().numpy()) for xi in x]
23
+ crows = torch.vstack([torch.from_numpy(xi.indptr) for xi in x])
24
+ cols = [torch.from_numpy(xi.indices) for xi in x]
25
+ max_cols = max(len(xi) for xi in cols)
26
+ cols = [
27
+ torch.cat([xi, pad + xi.new_zeros(max_cols - xi.shape[0])])
28
+ for xi in cols
29
+ ]
30
+ cols = torch.vstack(cols)
31
+ if dim == 2:
32
+ crows = crows[0]
33
+ cols = cols[0]
34
+ return crows.to(device), cols.to(device)
35
+
36
+
37
+def crow_col_to_dense(crows: torch.Tensor,
38
+ cols: torch.Tensor,
39
+ dtype: torch.dtype = torch.float16):
40
+ dim = crows.dim()
41
+ if dim == 1:
42
+ crows = crows[None]
43
+ cols = cols[None]
44
+ device = crows.device
45
+ crows, cols = crows.cpu(), cols.cpu() # faster in cpu
46
+ shape = (crows.shape[0], crows.shape[1] - 1, cols.max() + 1)
47
+ x = torch.zeros(shape, dtype=dtype)
48
+ for i in range(shape[0]):
49
+ for j in range(shape[1]):
50
+ x[i, j, cols[i, crows[i, j]:crows[i, j + 1]]] = 1
51
+ if dim == 1:
52
+ x = x[0]
53
+ return x.to(device)
54
+
55
+
56
+def dense_to_ccol_row(x: torch.Tensor):
57
+ """Similar, but to CSC format"""
58
+ x = x.transpose(-2, -1)
59
+ return dense_to_crow_col(x)
60
+
61
+
62
+def ccol_row_to_dense(ccol: torch.Tensor,
63
+ rows: torch.Tensor,
64
+ dtype: torch.dtype = torch.float16):
65
+ return crow_col_to_dense(ccol, rows, dtype).permute(0, 2, 1).contiguous()
66
+
67
+
68
+def _get_sparse_attn_mask_homo_head(
69
+ q_len: int,
70
+ max_seqlen: int,
71
+ dtype: torch.dtype,
72
+ device: torch.device,
73
+ block_size: int = 128,
74
+ local_blocks: int = 4,
75
+ vert_stride: int = 4,
76
+ return_dense: bool = False,
77
+):
78
+ """
79
+ :return: a tuple of 3:
80
+ - tuple of crow_indices, col_indices representation
81
+ of CSR format.
82
+ - block dense mask
83
+ - all token dense mask (be aware that it can be
84
+ OOM if it is too big) if `return_dense==True`,
85
+ otherwise, None
86
+ """
87
+ with torch.no_grad():
88
+ num_blocks = triton.cdiv(max_seqlen, block_size)
89
+ q_pos = torch.arange(num_blocks)[:, None]
90
+ k_pos = torch.arange(num_blocks)[None]
91
+ mask_vert_strided = (torch.arange(num_blocks) + 1) % vert_stride == 0
92
+ block_mask_dense = (((q_pos >= k_pos)
93
+ & ((q_pos - k_pos < local_blocks)
94
+ | mask_vert_strided)).to(device).to(dtype))
95
+ num_blocks_q = triton.cdiv(q_len, block_size)
96
+ block_mask_dense_output = (dense_to_crow_col(
97
+ block_mask_dense[-num_blocks_q:].contiguous()))
98
+ if return_dense:
99
+ mask_dense = torch.kron(
100
+ block_mask_dense,
101
+ block_mask_dense.new_ones((block_size, block_size)),
102
+ )
103
+ causal_mask = torch.tril(torch.ones(
104
+ max_seqlen, max_seqlen)).type_as(mask_dense)[-q_len:]
105
+ mask_dense = mask_dense[-q_len:, :max_seqlen] * causal_mask
106
+ return (
107
+ block_mask_dense_output,
108
+ block_mask_dense,
109
+ mask_dense,
110
+ )
111
+ else:
112
+ return (
113
+ block_mask_dense_output,
114
+ block_mask_dense,
115
+ None,
116
+ )
117
+
118
+
119
+def binary_mask_to_bias(mask_dense: torch.Tensor):
120
+ mask_dense = 1 - mask_dense
121
+ mask_dense.masked_fill_(mask_dense.bool(), -torch.inf)
122
+ return mask_dense
123
+
124
+
125
+def get_head_sliding_step(n_heads: int,
126
+ vert_stride: int,
127
+ homo_head: bool = False):
128
+ if homo_head:
129
+ return 0
130
+ return max(1, int(vert_stride / n_heads))
131
+
132
+
133
+@lru_cache
134
+def get_sparse_attn_mask(
135
+ n_heads: int,
136
+ q_len: int,
137
+ max_seqlen: int,
138
+ dtype: torch.dtype,
139
+ device: torch.device,
140
+ block_size: int = 64,
141
+ local_blocks: int = 4,
142
+ vert_stride: int = 4,
143
+ homo_head: bool = True,
144
+ return_dense: bool = False,
145
+ dense_mask_type: str = "binary",
146
+):
147
+ """
148
+ :param dense_mask_type: "binary" (0 for skip token, 1 for others)
149
+ or "bias" (-inf for skip token, 0 or others)
150
+ :return: a tuple of 3:
151
+ - tuple of crow_indices, col_indices representation
152
+ of CSR format.
153
+ - block dense mask
154
+ - all token dense mask (be aware that it can be OOM if it
155
+ is too big) if `return_dense==True`, otherwise, None
156
+ """
157
+ assert dense_mask_type in ("binary", "bias")
158
+ if homo_head:
159
+ with torch.no_grad():
160
+ (crow, col), block_mask_dense, mask_dense = (
161
+ _get_sparse_attn_mask_homo_head(
162
+ q_len,
163
+ max_seqlen,
164
+ dtype,
165
+ device,
166
+ block_size,
167
+ local_blocks,
168
+ vert_stride,
169
+ return_dense,
170
+ ))
171
+ crow = crow[None].expand(n_heads, crow.shape[0])
172
+ col = col[None].expand(n_heads, col.shape[0])
173
+ if return_dense:
174
+ mask_dense = mask_dense[None].expand(n_heads,
175
+ *mask_dense.shape)
176
+ if dense_mask_type == "bias":
177
+ mask_dense = binary_mask_to_bias(mask_dense)
178
+ return (crow, col), block_mask_dense, mask_dense
179
+
180
+ with torch.no_grad():
181
+ num_blocks = triton.cdiv(max_seqlen, block_size)
182
+ q_pos = torch.arange(num_blocks)[None, :, None]
183
+ k_pos = torch.arange(num_blocks)[None, None]
184
+ head_sliding_step = get_head_sliding_step(n_heads, vert_stride)
185
+ mask_vert_strided = [
186
+ (torch.arange(num_blocks) + h * head_sliding_step + 1) %
187
+ vert_stride == 0 for h in range(n_heads)
188
+ ]
189
+ mask_vert_strided = torch.vstack(mask_vert_strided).unsqueeze(1)
190
+ block_mask_dense = (((q_pos >= k_pos)
191
+ & ((q_pos - k_pos < local_blocks)
192
+ | mask_vert_strided)).to(device).to(dtype))
193
+ num_blocks_q = triton.cdiv(q_len, block_size)
194
+ block_mask_dense_output = block_mask_dense[:, -num_blocks_q:]
195
+ if return_dense:
196
+ mask_dense = torch.kron(
197
+ block_mask_dense,
198
+ block_mask_dense.new_ones((block_size, block_size)),
199
+ )
200
+ causal_mask = torch.tril(torch.ones(
201
+ max_seqlen, max_seqlen)).type_as(mask_dense)[-q_len:]
202
+ mask_dense = mask_dense[..., -q_len:, :max_seqlen] * causal_mask[None]
203
+ if dense_mask_type == "bias":
204
+ mask_dense = binary_mask_to_bias(mask_dense)
205
+
206
+ return (
207
+ dense_to_crow_col(block_mask_dense_output),
208
+ block_mask_dense,
209
+ mask_dense,
210
+ )
211
+ else:
212
+ return (
213
+ dense_to_crow_col(block_mask_dense_output),
214
+ block_mask_dense,
215
+ None,
216
+ )
vllm/attention/ops/paged_attn.py
+24
-1
@@ -91,9 +91,21 @@ class PagedAttention:
91
scale: float,
92
alibi_slopes: Optional[torch.Tensor],
93
kv_scale: float,
94
+ tp_rank: int = 0,
95
+ blocksparse_local_blocks: int = 0,
96
+ blocksparse_vert_stride: int = 0,
97
+ blocksparse_block_size: int = 64,
98
+ blocksparse_head_sliding_step: int = 0,
99
) -> torch.Tensor:
95
- output = torch.empty_like(query)
100
+ if blocksparse_vert_stride is not None and blocksparse_vert_stride > 1:
101
+ # use blocksparse paged attention
102
+ block_size = value_cache.size(-1)
103
+ assert (blocksparse_block_size > 0 and
104
+ blocksparse_block_size % block_size == 0), \
105
+ (f"{blocksparse_block_size=} needs to be a multiple of"
106
+ f"{block_size=} used in block_tables.")
107
108
+ output = torch.empty_like(query)
109
block_size = value_cache.shape[3]
110
num_seqs, num_heads, head_size = query.shape
111
max_num_partitions = ((max_seq_len + _PARTITION_SIZE - 1) //
@@ -107,6 +119,7 @@ class PagedAttention:
119
# For context len > 8192, use V2 kernel to avoid shared memory shortage.
120
use_v1 = (max_seq_len <= 8192
121
and (max_num_partitions == 1 or num_seqs * num_heads > 512))
122
+
123
if use_v1:
124
# Run PagedAttention V1.
125
ops.paged_attention_v1(
@@ -123,6 +136,11 @@ class PagedAttention:
136
alibi_slopes,
137
kv_cache_dtype,
138
kv_scale,
139
+ tp_rank,
140
+ blocksparse_local_blocks,
141
+ blocksparse_vert_stride,
142
+ blocksparse_block_size,
143
+ blocksparse_head_sliding_step,
144
)
145
else:
146
# Run PagedAttention V2.
@@ -155,6 +173,11 @@ class PagedAttention:
173
alibi_slopes,
174
kv_cache_dtype,
175
kv_scale,
176
+ tp_rank,
177
+ blocksparse_local_blocks,
178
+ blocksparse_vert_stride,
179
+ blocksparse_block_size,
180
+ blocksparse_head_sliding_step,
181
)
182
return output
183
vllm/attention/selector.py
+7
@@ -29,7 +29,14 @@ def get_attn_backend(
29
dtype: torch.dtype,
30
kv_cache_dtype: Optional[str],
31
block_size: int,
32
+ is_blocksparse: bool = False,
33
) -> Type[AttentionBackend]:
34
+
35
+ if is_blocksparse:
36
+ logger.info("Using BlocksparseFlashAttention backend.")
37
+ from vllm.attention.backends.blocksparse_attn import (
38
+ BlocksparseFlashAttentionBackend)
39
+ return BlocksparseFlashAttentionBackend
40
"""Determine which attention backend to use and only import
41
the selected backend module.
42
"""
vllm/entrypoints/openai/serving_engine.py
+1
@@ -100,6 +100,7 @@ class OpenAIServing:
100
token_logprob = step_top_logprobs[token_id].logprob
101
token = step_top_logprobs[token_id].decoded_token
102
logprobs.tokens.append(token)
103
+ token_logprob = max(token_logprob, -9999.0)
104
logprobs.token_logprobs.append(token_logprob)
105
106
if num_output_top_logprobs:
vllm/model_executor/models/__init__.py
+1
@@ -56,6 +56,7 @@ _GENERATION_MODELS = {
56
"Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
57
"ArcticForCausalLM": ("arctic", "ArcticForCausalLM"),
58
"XverseForCausalLM": ("xverse", "XverseForCausalLM"),
59
+ "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
60
}
61
62
_EMBEDDING_MODELS = {
vllm/model_executor/models/phi3_small.py
new
+447
@@ -0,0 +1,447 @@
1
+import math
2
+from typing import Iterable, List, Optional, Tuple
3
+
4
+import torch
5
+from torch import nn
6
+from transformers.configuration_utils import PretrainedConfig
7
+
8
+from vllm.attention import Attention, AttentionMetadata
9
+from vllm.config import CacheConfig, LoRAConfig
10
+from vllm.distributed import (get_tensor_model_parallel_rank,
11
+ get_tensor_model_parallel_world_size)
12
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
13
+ QKVParallelLinear,
14
+ RowParallelLinear)
15
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
16
+from vllm.model_executor.layers.quantization.base_config import (
17
+ QuantizationConfig)
18
+from vllm.model_executor.layers.rotary_embedding import get_rope
19
+from vllm.model_executor.layers.sampler import Sampler
20
+from vllm.model_executor.layers.vocab_parallel_embedding import (
21
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
22
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
+from vllm.model_executor.sampling_metadata import SamplingMetadata
24
+from vllm.sequence import SamplerOutput
25
+
26
+
27
+def load_column_parallel_weight(param: torch.nn.Parameter,
28
+ loaded_weight: torch.Tensor):
29
+ tp = get_tensor_model_parallel_world_size()
30
+ rk = get_tensor_model_parallel_rank()
31
+ assert param.size(0) * tp == loaded_weight.size(0)
32
+ s = rk * param.size(0)
33
+ e = (rk + 1) * param.size(0)
34
+ loaded_weight = loaded_weight[s:e]
35
+ assert param.shape == loaded_weight.shape
36
+ param.data.copy_(loaded_weight)
37
+
38
+
39
+class HeadMajorQKVParallelLinear(QKVParallelLinear):
40
+
41
+ def weight_loader(self, param: torch.nn.Parameter,
42
+ loaded_weight: torch.Tensor):
43
+ return load_column_parallel_weight(param, loaded_weight)
44
+
45
+
46
+class HeadMajorColumnParallelLinear(MergedColumnParallelLinear):
47
+
48
+ def weight_loader(self, param: torch.nn.Parameter,
49
+ loaded_weight: torch.Tensor):
50
+ return load_column_parallel_weight(param, loaded_weight)
51
+
52
+
53
+@torch.jit.script
54
+def quick_gelu(x):
55
+ return x * torch.sigmoid(1.702 * x)
56
+
57
+
58
+@torch.jit.script
59
+def gegelu(input, limit: Optional[float] = None):
60
+ a_gelu, a_linear = input[..., ::2], input[..., 1::2]
61
+ if limit is not None:
62
+ a_gelu = torch.where(torch.isinf(a_gelu), a_gelu,
63
+ a_gelu.clamp(min=None, max=limit))
64
+ a_linear = torch.where(
65
+ torch.isinf(a_linear),
66
+ a_linear,
67
+ a_linear.clamp(min=-limit, max=limit),
68
+ )
69
+ out_gelu = quick_gelu(a_gelu)
70
+ return out_gelu * (a_linear + 1)
71
+
72
+
73
+class Phi3SmallMLP(nn.Module):
74
+
75
+ def __init__(
76
+ self,
77
+ config: PretrainedConfig,
78
+ quant_config: Optional[QuantizationConfig] = None,
79
+ ) -> None:
80
+ super().__init__()
81
+ self.config = config
82
+ assert (self.config.hidden_act == "gegelu"
83
+ ), "Only `gegelu` is supported for the 4.7 series of models .."
84
+ self.hidden_size = config.hidden_size
85
+ self.gegelu_limit = config.gegelu_limit
86
+ self.intermediate_size = config.intermediate_size
87
+
88
+ self.up_proj = HeadMajorColumnParallelLinear(
89
+ self.hidden_size,
90
+ 2 * [self.intermediate_size],
91
+ bias=True,
92
+ quant_config=quant_config,
93
+ )
94
+ self.down_proj = RowParallelLinear(
95
+ self.intermediate_size,
96
+ self.hidden_size,
97
+ bias=True,
98
+ quant_config=quant_config,
99
+ )
100
+
101
+ def forward(self, x):
102
+ gate_up, _ = self.up_proj(x)
103
+ x = gegelu(gate_up)
104
+ x, _ = self.down_proj(x)
105
+ return x
106
+
107
+
108
+class Phi3SmallSelfAttention(nn.Module):
109
+
110
+ def __init__(
111
+ self,
112
+ config: PretrainedConfig,
113
+ layer_idx: int,
114
+ cache_config: Optional[CacheConfig] = None,
115
+ quant_config: Optional[QuantizationConfig] = None,
116
+ ) -> None:
117
+ super().__init__()
118
+ self.layer_idx = layer_idx
119
+ self.config = config
120
+ self.sparse_block_size = config.blocksparse_block_size
121
+ self.homo_heads = config.blocksparse_homo_head_pattern
122
+ self.local_blocks = config.blocksparse_num_local_blocks
123
+ self.vert_stride = config.blocksparse_vert_stride
124
+
125
+ assert (config.blocksparse_block_size ==
126
+ config.blocksparse_triton_kernel_block_size)
127
+
128
+ self.hidden_size = config.hidden_size
129
+ # Number of Query Heads
130
+ self.num_heads = config.num_attention_heads
131
+
132
+ self.head_dim = self.hidden_size // self.num_heads
133
+ self.tp_size = get_tensor_model_parallel_world_size()
134
+ # Number of total Key Value Heads before tensor parallel
135
+ self.num_key_value_heads = config.num_key_value_heads
136
+ self.num_q_per_kv = self.num_heads // self.num_key_value_heads
137
+ if self.tp_size > 1:
138
+ assert self.num_key_value_heads % self.tp_size == 0
139
+ self.num_kv_heads_per_partion = max(
140
+ 1, self.num_key_value_heads // self.tp_size)
141
+ self.num_heads_per_partition = self.num_heads // self.tp_size
142
+
143
+ self.max_position_embeddings = config.max_position_embeddings
144
+ self.rope_embedding_base = config.rope_embedding_base
145
+ self.rope_position_scale = config.rope_position_scale
146
+ self.is_causal = True
147
+
148
+ norm_factor = None
149
+ if config.mup_use_scaling:
150
+ norm_factor = self.head_dim / config.mup_attn_multiplier
151
+ else:
152
+ norm_factor = math.sqrt(self.head_dim)
153
+ self.scale = 1 / norm_factor
154
+
155
+ self.query_key_value = HeadMajorQKVParallelLinear(
156
+ self.hidden_size,
157
+ self.head_dim,
158
+ self.num_heads,
159
+ self.num_key_value_heads,
160
+ bias=True,
161
+ quant_config=quant_config,
162
+ )
163
+
164
+ self.dense = RowParallelLinear(self.hidden_size,
165
+ self.hidden_size,
166
+ bias=True,
167
+ quant_config=quant_config)
168
+
169
+ if getattr(self.config, "rope_scaling", None) is not None:
170
+ rope_scaling = self.config.rope_scaling
171
+ for key in rope_scaling:
172
+ if isinstance(rope_scaling[key], list):
173
+ rope_scaling[key] = tuple(rope_scaling[key])
174
+
175
+ if "factor" not in rope_scaling:
176
+ rope_scaling["factor"] = self.rope_position_scale
177
+ else:
178
+ rope_scaling = {
179
+ "type": "linear",
180
+ "factor": self.rope_position_scale,
181
+ }
182
+
183
+ self.rotary_emb = get_rope(
184
+ self.head_dim,
185
+ rotary_dim=self.head_dim,
186
+ max_position=self.max_position_embeddings,
187
+ base=self.rope_embedding_base,
188
+ rope_scaling=rope_scaling,
189
+ )
190
+
191
+ # blocksparse params
192
+ self.blocksparse_block_size = config.blocksparse_block_size
193
+ self.blocksparse_num_local_blocks = config.blocksparse_num_local_blocks
194
+ self.blocksparse_vert_stride = config.blocksparse_vert_stride
195
+
196
+ use_dense_attn = (getattr(self.config,
197
+ "dense_attention_every_n_layers", None)
198
+ and (self.layer_idx + 1) %
199
+ self.config.dense_attention_every_n_layers == 0)
200
+
201
+ bs_params = None
202
+ if not use_dense_attn:
203
+ bs_params = {
204
+ 'max_seqlen': self.max_position_embeddings,
205
+ 'num_heads': self.num_heads_per_partition,
206
+ "num_kv_heads": self.num_kv_heads_per_partion,
207
+ "block_size": self.sparse_block_size,
208
+ "local_blocks": self.local_blocks,
209
+ "vert_stride": self.vert_stride,
210
+ "homo_head": self.homo_heads
211
+ }
212
+
213
+ self.attn = Attention(
214
+ self.num_heads_per_partition,
215
+ self.head_dim,
216
+ self.scale,
217
+ num_kv_heads=self.num_kv_heads_per_partion,
218
+ cache_config=cache_config,
219
+ quant_config=quant_config,
220
+ blocksparse_params=bs_params,
221
+ )
222
+
223
+ def forward(
224
+ self,
225
+ positions: torch.Tensor,
226
+ hidden_states: torch.Tensor,
227
+ kv_cache: torch.Tensor,
228
+ attn_metadata: AttentionMetadata,
229
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
230
+ Optional[Tuple[torch.Tensor]]]:
231
+ qkv, _ = self.query_key_value(hidden_states)
232
+
233
+ qkv = qkv.view(qkv.shape[:-1] +
234
+ (-1, (self.num_q_per_kv + 2), self.head_dim))
235
+ q, k, v = qkv.split([self.num_q_per_kv, 1, 1], dim=-2)
236
+
237
+ # NOTE: this is required by RotaryEmbed, which indeed does not have to
238
+ # TODO: allow 3D QK for rotary forward
239
+ q = q.reshape(-1, self.head_dim * self.num_heads_per_partition)
240
+ k = k.reshape(-1, self.head_dim * self.num_kv_heads_per_partion)
241
+ v = v.reshape(-1, self.head_dim * self.num_kv_heads_per_partion)
242
+
243
+ q, k = self.rotary_emb(positions, q, k)
244
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata=attn_metadata)
245
+ output, _ = self.dense(attn_output)
246
+
247
+ return output
248
+
249
+
250
+class Phi3SmallDecoderLayer(nn.Module):
251
+
252
+ def __init__(
253
+ self,
254
+ config: PretrainedConfig,
255
+ layer_idx: int,
256
+ cache_config: Optional[CacheConfig] = None,
257
+ quant_config: Optional[QuantizationConfig] = None,
258
+ ):
259
+ super().__init__()
260
+ self.hidden_size = config.hidden_size
261
+ self.self_attn = Phi3SmallSelfAttention(config,
262
+ layer_idx,
263
+ cache_config=cache_config,
264
+ quant_config=quant_config)
265
+ self.mlp = Phi3SmallMLP(config, quant_config)
266
+
267
+ self.input_layernorm = nn.LayerNorm(config.hidden_size,
268
+ eps=config.layer_norm_epsilon)
269
+ self.post_attention_layernorm = nn.LayerNorm(
270
+ config.hidden_size, eps=config.layer_norm_epsilon)
271
+
272
+ def forward(
273
+ self,
274
+ positions: torch.Tensor,
275
+ hidden_states: torch.Tensor,
276
+ kv_cache: torch.Tensor,
277
+ attn_metadata: AttentionMetadata,
278
+ ) -> torch.Tensor:
279
+ residual = hidden_states
280
+ hidden_states = self.input_layernorm(hidden_states)
281
+
282
+ hidden_states = self.self_attn(
283
+ positions=positions,
284
+ hidden_states=hidden_states,
285
+ kv_cache=kv_cache,
286
+ attn_metadata=attn_metadata,
287
+ )
288
+ hidden_states = residual + hidden_states
289
+
290
+ residual = hidden_states
291
+ hidden_states = self.post_attention_layernorm(hidden_states)
292
+ hidden_states = self.mlp(hidden_states)
293
+ hidden_states = residual + hidden_states
294
+ return hidden_states
295
+
296
+
297
+class Phi3SmallModel(nn.Module):
298
+
299
+ def __init__(
300
+ self,
301
+ config: PretrainedConfig,
302
+ cache_config: Optional[CacheConfig] = None,
303
+ quant_config: Optional[QuantizationConfig] = None,
304
+ ):
305
+ super().__init__()
306
+ self.config = config
307
+ self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
308
+ config.hidden_size)
309
+ self.mup_embedding_multiplier = config.mup_embedding_multiplier
310
+ self.layers = nn.ModuleList([
311
+ Phi3SmallDecoderLayer(config, layer_idx, cache_config,
312
+ quant_config)
313
+ for layer_idx in range(config.num_hidden_layers)
314
+ ])
315
+
316
+ self.final_layernorm = nn.LayerNorm(config.hidden_size,
317
+ eps=config.layer_norm_epsilon)
318
+
319
+ def get_input_embeddings(self):
320
+ return self.embed_tokens
321
+
322
+ def set_input_embeddings(self, value):
323
+ self.embed_tokens = value
324
+
325
+ def forward(
326
+ self,
327
+ input_ids: torch.LongTensor,
328
+ positions: Optional[torch.LongTensor],
329
+ kv_caches: List[torch.Tensor],
330
+ attn_metadata: AttentionMetadata = None,
331
+ ):
332
+ hidden_states = self.embed_tokens(input_ids)
333
+ if (self.mup_embedding_multiplier is not None
334
+ and self.mup_embedding_multiplier > 0.0):
335
+ hidden_states = hidden_states * self.mup_embedding_multiplier
336
+ for i in range(len(self.layers)):
337
+ layer = self.layers[i]
338
+ hidden_states = layer(
339
+ positions,
340
+ hidden_states,
341
+ kv_caches[i],
342
+ attn_metadata,
343
+ )
344
+ hidden_states = self.final_layernorm(hidden_states)
345
+ return hidden_states
346
+
347
+
348
+class Phi3SmallForCausalLM(nn.Module):
349
+ _tied_weights_keys = ["lm_head.weight"]
350
+
351
+ def __init__(
352
+ self,
353
+ config,
354
+ cache_config: Optional[CacheConfig] = None,
355
+ quant_config: Optional[QuantizationConfig] = None,
356
+ lora_config: Optional[LoRAConfig] = None,
357
+ ):
358
+ super().__init__()
359
+ self.config = config
360
+ self.quant_config = quant_config
361
+ self.model = Phi3SmallModel(config, cache_config, quant_config)
362
+ self.vocab_size = config.vocab_size
363
+ self.mup_width_multiplier = config.mup_width_multiplier
364
+ self.lm_head = ParallelLMHead(
365
+ self.vocab_size,
366
+ config.hidden_size,
367
+ org_num_embeddings=config.vocab_size,
368
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE,
369
+ )
370
+ self.logits_processor = LogitsProcessor(config.vocab_size)
371
+ self.sampler = Sampler()
372
+
373
+ # tokens in tiktoken but not used
374
+ if hasattr(config, 'dummy_token_indices'):
375
+ device = self.lm_head.weight.device
376
+ self.register_buffer('dummy_token_indices',
377
+ torch.LongTensor(
378
+ config.dummy_token_indices).to(device),
379
+ persistent=False)
380
+ else:
381
+ self.dummy_token_indices = None
382
+
383
+ def get_input_embeddings(self):
384
+ return self.model.embed_tokens
385
+
386
+ def set_input_embeddings(self, value):
387
+ self.model.embed_tokens = value
388
+
389
+ def get_output_embeddings(self):
390
+ return self.lm_head
391
+
392
+ def set_output_embeddings(self, value):
393
+ self.lm_head = value
394
+
395
+ def set_decoder(self, decoder):
396
+ self.model = decoder
397
+
398
+ def get_decoder(self):
399
+ return self.model
400
+
401
+ def compute_logits(self, hidden_states: torch.Tensor,
402
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
403
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
404
+ sampling_metadata)
405
+ if self.dummy_token_indices is not None and logits is not None:
406
+ logits.index_fill_(-1, self.dummy_token_indices, -torch.inf)
407
+ return logits
408
+
409
+ def forward(
410
+ self,
411
+ input_ids: torch.LongTensor,
412
+ positions: Optional[torch.LongTensor],
413
+ kv_caches: List[torch.Tensor],
414
+ attn_metadata: AttentionMetadata,
415
+ ) -> torch.Tensor:
416
+ output_hidden_states = self.model(
417
+ input_ids=input_ids,
418
+ positions=positions,
419
+ kv_caches=kv_caches,
420
+ attn_metadata=attn_metadata,
421
+ )
422
+ output_hidden_states = output_hidden_states
423
+ return output_hidden_states
424
+
425
+ def sample(
426
+ self,
427
+ logits: torch.Tensor,
428
+ sampling_metadata: SamplingMetadata,
429
+ ) -> Optional[SamplerOutput]:
430
+
431
+ next_tokens = self.sampler(logits / self.mup_width_multiplier,
432
+ sampling_metadata)
433
+ return next_tokens
434
+
435
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
436
+
437
+ params_dict = dict(self.named_parameters())
438
+ for name, loaded_weight in weights:
439
+ if "rotary_emb.inv_freq" in name:
440
+ continue
441
+ if name.endswith(".bias") and name not in params_dict:
442
+ continue
443
+ param = params_dict[name]
444
+ weight_loader = getattr(param, "weight_loader",
445
+ default_weight_loader)
446
+ weight_loader(param, loaded_weight)
447
+ self.lm_head.weight.data.copy_(self.model.embed_tokens.weight.data)
vllm/transformers_utils/config.py
+1
-1
@@ -63,4 +63,4 @@ def get_hf_text_config(config: PretrainedConfig):
63
assert hasattr(config.text_config, "num_attention_heads")
64
return config.text_config
65
else:
66
- return config
66
+ return config
\ No newline at end of file