A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
[FIX] Fix Alibi implementation in PagedAttention kernel (#945)
* [FIX] Fix Alibi implementation in PagedAttention kernel * Fix test_attention * Fix --------- Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Oliver-ss <yuansongwx@outlook.com>
Zhuohan Li committed
Sep 7, 2023 at 15:53 UTC
db09d4ad833b1d8911f14852a484790c53818a5d
2 files changed
+4
-3
csrc/attention/attention_kernels.cu
+1
-1
@@ -178,7 +178,7 @@ __global__ void single_query_cached_kv_attention_kernel(
178
// This includes a reduction across the threads in the same thread group.
179
float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs[thread_group_offset], k_vecs);
180
// Add the ALiBi bias if slopes are given.
181
- qk += (alibi_slope != 0) ? alibi_slope * (token_idx - context_len) : 0;
181
+ qk += (alibi_slope != 0) ? alibi_slope * (token_idx - context_len + 1) : 0;
182
183
if (thread_group_offset == 0) {
184
// Store the partial reductions to shared memory.
tests/kernels/test_attention.py
+3
-2
@@ -17,7 +17,7 @@ NUM_PREFILL_SEQS = [1, 3, 7] # Arbitrary values for testing
17
NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
18
HEAD_SIZES = [64, 80, 96, 112, 128, 256]
19
BLOCK_SIZES = [8, 16, 32]
20
-USE_ALIBI = [False] # TODO(woosuk): Add USE_ALIBI=True
20
+USE_ALIBI = [False, True]
21
SEEDS = [0]
22
23
@@ -83,7 +83,7 @@ def ref_single_query_cached_kv_attention(
83
if alibi_slopes is not None:
84
# Create the ALiBi bias used in the paged attention kernel.
85
position_ids = torch.arange(context_len, device="cuda").int()
86
- alibi_bias = (context_len - position_ids).float()
86
+ alibi_bias = (position_ids - context_len + 1).float()
87
alibi_bias = alibi_slopes.view(-1, 1, 1) * alibi_bias.view(
88
1, 1, -1)
89
@@ -224,6 +224,7 @@ def ref_multi_query_kv_attention(
224
return ref_output
225
226
227
+# TODO(woosuk): Add tests for USE_ALIBI=True.
228
@pytest.mark.parametrize("num_seqs", NUM_PREFILL_SEQS)
229
@pytest.mark.parametrize("num_heads", NUM_HEADS)
230
@pytest.mark.parametrize("head_size", HEAD_SIZES)