@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / db09d4ad83
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

[FIX] Fix Alibi implementation in PagedAttention kernel (#945)

* [FIX] Fix Alibi implementation in PagedAttention kernel * Fix test_attention * Fix --------- Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: Oliver-ss <yuansongwx@outlook.com>

Zhuohan Li committed Sep 7, 2023 at 15:53 UTC db09d4ad833b1d8911f14852a484790c53818a5d
2 files changed +4 -3
csrc/attention/attention_kernels.cu
+1 -1
@@ -178,7 +178,7 @@ __global__ void single_query_cached_kv_attention_kernel(
178 // This includes a reduction across the threads in the same thread group.
179 float qk = scale * Qk_dot<scalar_t, THREAD_GROUP_SIZE>::dot(q_vecs[thread_group_offset], k_vecs);
180 // Add the ALiBi bias if slopes are given.
181 - qk += (alibi_slope != 0) ? alibi_slope * (token_idx - context_len) : 0;
181 + qk += (alibi_slope != 0) ? alibi_slope * (token_idx - context_len + 1) : 0;
182
183 if (thread_group_offset == 0) {
184 // Store the partial reductions to shared memory.
tests/kernels/test_attention.py
+3 -2
@@ -17,7 +17,7 @@ NUM_PREFILL_SEQS = [1, 3, 7] # Arbitrary values for testing
17 NUM_HEADS = [(40, 40), (64, 8)] # Arbitrary values for testing
18 HEAD_SIZES = [64, 80, 96, 112, 128, 256]
19 BLOCK_SIZES = [8, 16, 32]
20 -USE_ALIBI = [False] # TODO(woosuk): Add USE_ALIBI=True
20 +USE_ALIBI = [False, True]
21 SEEDS = [0]
22
23
@@ -83,7 +83,7 @@ def ref_single_query_cached_kv_attention(
83 if alibi_slopes is not None:
84 # Create the ALiBi bias used in the paged attention kernel.
85 position_ids = torch.arange(context_len, device="cuda").int()
86 - alibi_bias = (context_len - position_ids).float()
86 + alibi_bias = (position_ids - context_len + 1).float()
87 alibi_bias = alibi_slopes.view(-1, 1, 1) * alibi_bias.view(
88 1, 1, -1)
89
@@ -224,6 +224,7 @@ def ref_multi_query_kv_attention(
224 return ref_output
225
226
227 +# TODO(woosuk): Add tests for USE_ALIBI=True.
228 @pytest.mark.parametrize("num_seqs", NUM_PREFILL_SEQS)
229 @pytest.mark.parametrize("num_heads", NUM_HEADS)
230 @pytest.mark.parametrize("head_size", HEAD_SIZES)