@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 32881f3f31
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[kernel] fix sliding window in prefix prefill Triton kernel (#4405)

Co-authored-by: SangBin Cho <rkooo567@gmail.com>

Michał Moskal committed May 2, 2024 at 11:23 UTC 32881f3f3106e17d2fd52d8ac00217a0f0b2476a
6 files changed +91 -23
tests/kernels/test_prefix_prefill.py
+30 -4
@@ -15,6 +15,7 @@ DTYPES = [torch.float16]
15 CUDA_DEVICES = [
16 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
17 ]
18 +SLIDING_WINDOW = [0, 16, 64, 128, 256, 512, 2048]
19
20
21 @pytest.mark.parametrize("num_heads", NUM_HEADS)
@@ -22,11 +23,13 @@ CUDA_DEVICES = [
23 @pytest.mark.parametrize("head_size", HEAD_SIZES)
24 @pytest.mark.parametrize("dtype", DTYPES)
25 @pytest.mark.parametrize("device", CUDA_DEVICES)
26 +@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
27 @torch.inference_mode()
28 def test_contexted_kv_attention(
29 num_heads: int,
30 num_queries_per_kv: int,
31 head_size: int,
32 + sliding_window: int,
33 dtype: torch.dtype,
34 device: str,
35 ) -> None:
@@ -123,12 +126,32 @@ def test_contexted_kv_attention(
126
127 # Warm up the Triton kernel by calling it once before actually measuring
128 # generation time
126 - context_attention_fwd(query, k, v, output, k_cache, v_cache, block_table,
127 - b_start_loc, b_seq_len, b_ctx_len, max_input_len)
129 + context_attention_fwd(query,
130 + k,
131 + v,
132 + output,
133 + k_cache,
134 + v_cache,
135 + block_table,
136 + b_start_loc,
137 + b_seq_len,
138 + b_ctx_len,
139 + max_input_len,
140 + sliding_window=sliding_window)
141 torch.cuda.synchronize()
142 start_time = time.time()
130 - context_attention_fwd(query, k, v, output, k_cache, v_cache, block_table,
131 - b_start_loc, b_seq_len, b_ctx_len, max_input_len)
143 + context_attention_fwd(query,
144 + k,
145 + v,
146 + output,
147 + k_cache,
148 + v_cache,
149 + block_table,
150 + b_start_loc,
151 + b_seq_len,
152 + b_ctx_len,
153 + max_input_len,
154 + sliding_window=sliding_window)
155 torch.cuda.synchronize()
156 end_time = time.time()
157 print(f"triton Time: {(end_time - start_time)*1000:.2f} ms")
@@ -156,6 +179,9 @@ def test_contexted_kv_attention(
179
180 attn_bias = BlockDiagonalCausalFromBottomRightMask.from_seqlens(
181 subquery_lens, seq_lens)
182 + if sliding_window > 0:
183 + attn_bias = attn_bias.make_local_attention_from_bottomright(
184 + sliding_window)
185 output_ref = xops.memory_efficient_attention_forward(
186 query,
187 key,
vllm/attention/backends/flash_attn.py
+1
@@ -249,6 +249,7 @@ class FlashAttentionImpl(AttentionImpl):
249 prefill_meta.context_lens,
250 prefill_meta.max_subquery_len,
251 self.alibi_slopes,
252 + self.sliding_window[0],
253 )
254 if decode_meta := attn_metadata.decode_metadata:
255 # Decoding run.
vllm/attention/backends/rocm_flash_attn.py
+1
@@ -307,6 +307,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
307 prefill_meta.context_lens,
308 prefill_meta.max_subquery_len,
309 self.alibi_slopes,
310 + self.sliding_window[0],
311 )
312
313 if decode_meta := attn_metadata.decode_metadata:
vllm/attention/backends/xformers.py
+1
@@ -246,6 +246,7 @@ class XFormersImpl(AttentionImpl):
246 prefill_meta.context_lens,
247 prefill_meta.max_subquery_len,
248 self.alibi_slopes,
249 + self.sliding_window,
250 )
251 assert output[:num_prefill_tokens].shape == out.shape
252 output[:num_prefill_tokens] = out
vllm/attention/ops/paged_attn.py
+2
@@ -172,6 +172,7 @@ class PagedAttention:
172 context_lens: torch.Tensor,
173 max_subquery_len: int,
174 alibi_slopes: Optional[torch.Tensor],
175 + sliding_window: Optional[int],
176 ) -> torch.Tensor:
177 output = torch.empty_like(query)
178 context_attention_fwd(
@@ -188,6 +189,7 @@ class PagedAttention:
189 context_lens,
190 max_subquery_len,
191 alibi_slopes,
192 + sliding_window,
193 )
194 return output
195
vllm/attention/ops/prefix_prefill.py
+56 -19
@@ -50,6 +50,7 @@ if triton.__version__ >= "2.1.0":
50 BLOCK_DMODEL: tl.constexpr, # head size
51 BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2
52 BLOCK_N: tl.constexpr,
53 + SLIDING_WINDOW: tl.constexpr,
54 ):
55 cur_batch = tl.program_id(0)
56 cur_head = tl.program_id(1)
@@ -62,42 +63,53 @@ if triton.__version__ >= "2.1.0":
63 cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
64 cur_batch_query_len = cur_batch_seq_len - cur_batch_ctx_len
65
66 + # start position inside of the query
67 + # generally, N goes over kv, while M goes over query_len
68 block_start_loc = BLOCK_M * start_m
69
70 # initialize offsets
71 + # [N]; starts at 0
72 offs_n = tl.arange(0, BLOCK_N)
73 + # [D]; starts at 0
74 offs_d = tl.arange(0, BLOCK_DMODEL_PADDED)
75 + # [M]; starts at current position in query
76 offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
77 + # [M,D]
78 off_q = (
79 (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs +
80 cur_head * stride_qh + offs_d[None, :] * stride_qd)
81
82 dim_mask = tl.where(
76 - tl.arange(0, BLOCK_DMODEL_PADDED) < BLOCK_DMODEL, 1, 0).to(tl.int1)
83 + tl.arange(0, BLOCK_DMODEL_PADDED) < BLOCK_DMODEL, 1,
84 + 0).to(tl.int1) # [D]
85
86 q = tl.load(Q + off_q,
87 mask=dim_mask[None, :] &
88 (offs_m[:, None] < cur_batch_query_len),
81 - other=0.0)
89 + other=0.0) # [M,D]
90
83 - # # initialize pointer to m and l
84 - m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
85 - l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
86 - acc = tl.zeros([BLOCK_M, BLOCK_DMODEL_PADDED], dtype=tl.float32)
91 + # initialize pointer to m and l
92 + m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf") # [M]
93 + l_i = tl.zeros([BLOCK_M], dtype=tl.float32) # [M]
94 + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL_PADDED],
95 + dtype=tl.float32) # [M,D]
96
97 + # compute query against context (no causal mask here)
98 for start_n in range(0, cur_batch_ctx_len, BLOCK_N):
99 start_n = tl.multiple_of(start_n, BLOCK_N)
100 # -- compute qk ----
101 bn = tl.load(B_Loc + cur_batch * stride_b_loc_b +
102 ((start_n + offs_n) // block_size) * stride_b_loc_s,
103 mask=(start_n + offs_n) < cur_batch_ctx_len,
94 - other=0)
104 + other=0) # [N]
105 + # [D,N]
106 off_k = (bn[None, :] * stride_k_cache_bs +
107 cur_kv_head * stride_k_cache_h +
108 (offs_d[:, None] // x) * stride_k_cache_d +
109 ((start_n + offs_n[None, :]) % block_size) *
110 stride_k_cache_bl +
111 (offs_d[:, None] % x) * stride_k_cache_x)
112 + # [N,D]
113 off_v = (
114 bn[:, None] * stride_v_cache_bs +
115 cur_kv_head * stride_v_cache_h +
@@ -106,23 +118,39 @@ if triton.__version__ >= "2.1.0":
118 k = tl.load(K_cache + off_k,
119 mask=dim_mask[:, None] &
120 ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
109 - other=0.0)
121 + other=0.0) # [D,N]
122
111 - qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
123 + qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32) # [M,N]
124 qk += tl.dot(q, k)
125 qk = tl.where((start_n + offs_n[None, :]) < cur_batch_ctx_len, qk,
126 float("-inf"))
127 qk *= sm_scale
128 + if SLIDING_WINDOW > 0:
129 + # (cur_batch_ctx_len + offs_m[:, None]) are the positions of
130 + # Q entries in sequence
131 + # (start_n + offs_n[None, :]) are the positions of
132 + # KV entries in sequence
133 + # So the condition makes sure each entry in Q only attends
134 + # to KV entries not more than SLIDING_WINDOW away.
135 + #
136 + # We can't use -inf here, because the
137 + # sliding window may lead to the entire row being masked.
138 + # This then makes m_ij contain -inf, which causes NaNs in
139 + # exp().
140 + qk = tl.where((cur_batch_ctx_len + offs_m[:, None]) -
141 + (start_n + offs_n[None, :]) < SLIDING_WINDOW, qk,
142 + -10000)
143
144 # -- compute m_ij, p, l_ij
118 - m_ij = tl.max(qk, 1)
119 - p = tl.exp(qk - m_ij[:, None])
120 - l_ij = tl.sum(p, 1)
145 + m_ij = tl.max(qk, 1) # [M]
146 + p = tl.exp(qk - m_ij[:, None]) # [M,N]
147 + l_ij = tl.sum(p, 1) # [M]
148 # -- update m_i and l_i
122 - m_i_new = tl.maximum(m_i, m_ij)
123 - alpha = tl.exp(m_i - m_i_new)
124 - beta = tl.exp(m_ij - m_i_new)
125 - l_i_new = alpha * l_i + beta * l_ij
149 + m_i_new = tl.maximum(m_i, m_ij) # [M]
150 + alpha = tl.exp(m_i - m_i_new) # [M]
151 + beta = tl.exp(m_ij - m_i_new) # [M]
152 + l_i_new = alpha * l_i + beta * l_ij # [M]
153 +
154 # -- update output accumulator --
155 # scale p
156 p_scale = beta / l_i_new
@@ -134,7 +162,7 @@ if triton.__version__ >= "2.1.0":
162 v = tl.load(V_cache + off_v,
163 mask=dim_mask[None, :] &
164 ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
137 - other=0.0)
165 + other=0.0) # [N,D]
166
167 p = p.to(v.dtype)
168 acc += tl.dot(p, v)
@@ -149,8 +177,10 @@ if triton.__version__ >= "2.1.0":
177 k_ptrs = K + off_k
178 v_ptrs = V + off_v
179
180 + # block_mask is 0 when we're already past the current query length
181 block_mask = tl.where(block_start_loc < cur_batch_query_len, 1, 0)
182
183 + # compute query against itself (with causal mask)
184 for start_n in range(0, block_mask * (start_m + 1) * BLOCK_M, BLOCK_N):
185 start_n = tl.multiple_of(start_n, BLOCK_N)
186 # -- compute qk ----
@@ -163,8 +193,13 @@ if triton.__version__ >= "2.1.0":
193 qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
194 qk += tl.dot(q, k)
195 qk *= sm_scale
196 + # apply causal mask
197 qk = tl.where(offs_m[:, None] >= (start_n + offs_n[None, :]), qk,
198 float("-inf"))
199 + if SLIDING_WINDOW > 0:
200 + qk = tl.where(
201 + offs_m[:, None] -
202 + (start_n + offs_n[None, :]) < SLIDING_WINDOW, qk, -10000)
203
204 # -- compute m_ij, p, l_ij
205 m_ij = tl.max(qk, 1)
@@ -636,7 +671,8 @@ if triton.__version__ >= "2.1.0":
671 b_seq_len,
672 b_ctx_len,
673 max_input_len,
639 - alibi_slopes=None):
674 + alibi_slopes=None,
675 + sliding_window=None):
676
677 cap = torch.cuda.get_device_capability()
678 BLOCK = 128 if cap[0] >= 8 else 64
@@ -644,7 +680,7 @@ if triton.__version__ >= "2.1.0":
680 Lq, Lk, Lv = q.shape[-1], k.shape[-1], v.shape[-1]
681 assert Lq == Lk and Lk == Lv
682 # round up Lk to a power of 2 - this is required for Triton block size
647 - Lk_padded = 2**((Lk - 1).bit_length())
683 + Lk_padded = triton.next_power_of_2(Lk)
684
685 sm_scale = 1.0 / (Lq**0.5)
686 batch, head = b_seq_len.shape[0], q.shape[1]
@@ -749,6 +785,7 @@ if triton.__version__ >= "2.1.0":
785 BLOCK_DMODEL=Lk,
786 BLOCK_DMODEL_PADDED=Lk_padded,
787 BLOCK_N=BLOCK,
788 + SLIDING_WINDOW=sliding_window if sliding_window is not None else 0,
789 num_warps=num_warps,
790 num_stages=1,
791 )