@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 71bcaf99e2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Enable GQA support in the prefix prefill kernels (#3007)

Signed-off-by: Tao He <sighingnow@gmail.com>

Tao He committed Feb 27, 2024 at 17:14 UTC 71bcaf99e2cb2c677bf3a9addb9e8039cbcab22a
3 files changed +87 -47
tests/kernels/test_prefix_prefill.py
+42 -19
@@ -8,7 +8,8 @@ from vllm.model_executor.layers.triton_kernel.prefix_prefill import (
8 from xformers import ops as xops
9 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
10
11 -NUM_HEADS = [12]
11 +NUM_HEADS = [64]
12 +NUM_QUERIES_PER_KV = [1, 8, 64]
13 HEAD_SIZES = [128]
14 DTYPES = [torch.float16]
15 CUDA_DEVICES = [
@@ -17,12 +18,14 @@ CUDA_DEVICES = [
18
19
20 @pytest.mark.parametrize("num_heads", NUM_HEADS)
21 +@pytest.mark.parametrize("num_queries_per_kv", NUM_HEADS)
22 @pytest.mark.parametrize("head_size", HEAD_SIZES)
23 @pytest.mark.parametrize("dtype", DTYPES)
24 @pytest.mark.parametrize("device", CUDA_DEVICES)
25 @torch.inference_mode()
26 def test_contexted_kv_attention(
27 num_heads: int,
28 + num_queries_per_kv: int,
29 head_size: int,
30 dtype: torch.dtype,
31 device: str,
@@ -41,28 +44,29 @@ def test_contexted_kv_attention(
44 subquery_lens = [random.randint(16, MAX_SEQ_LEN) for _ in range(BS)]
45 ctx_lens = [random.randint(16, MAX_CTX_LEN) for _ in range(BS)]
46 seq_lens = [a + b for a, b in zip(subquery_lens, ctx_lens)]
47 + num_kv_heads = num_heads // num_queries_per_kv
48
49 num_tokens = sum(subquery_lens)
50 query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
51 query.uniform_(-1e-3, 1e-3)
52 output = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
53
50 - kv = torch.empty(sum(seq_lens), 2, num_heads, head_size, dtype=dtype)
54 + kv = torch.empty(sum(seq_lens), 2, num_kv_heads, head_size, dtype=dtype)
55 kv.uniform_(-1e-3, 1e-3)
56 key, value = kv.unbind(dim=1)
57
58 k_cache = torch.zeros(cache_size,
59 block_size,
56 - num_heads,
60 + num_kv_heads,
61 head_size,
62 dtype=dtype)
63 v_cache = torch.zeros(cache_size,
64 block_size,
61 - num_heads,
65 + num_kv_heads,
66 head_size,
67 dtype=dtype)
64 - k = torch.zeros(sum(subquery_lens), num_heads, head_size, dtype=dtype)
65 - v = torch.zeros(sum(subquery_lens), num_heads, head_size, dtype=dtype)
68 + k = torch.zeros(sum(subquery_lens), num_kv_heads, head_size, dtype=dtype)
69 + v = torch.zeros(sum(subquery_lens), num_kv_heads, head_size, dtype=dtype)
70 values = torch.arange(0, cache_size, dtype=torch.long)
71 values = values[torch.randperm(cache_size)]
72 block_table = values[:BS * max_block_per_request].view(
@@ -93,19 +97,21 @@ def test_contexted_kv_attention(
97 end_loc = start_loc + block_size
98 start_slot = block_table[i, block_id] * block_size
99 end_slot = start_slot + end_loc - start_loc
96 - k_cache.view(-1, num_heads, head_size)[start_slot:end_slot].copy_(
97 - key[start_loc:end_loc])
98 - v_cache.view(-1, num_heads, head_size)[start_slot:end_slot].copy_(
99 - value[start_loc:end_loc])
100 + k_cache.view(-1, num_kv_heads,
101 + head_size)[start_slot:end_slot].copy_(
102 + key[start_loc:end_loc])
103 + v_cache.view(-1, num_kv_heads,
104 + head_size)[start_slot:end_slot].copy_(
105 + value[start_loc:end_loc])
106 cur_ctx += block_size
107 block_id += 1
108 # transpose K_cache[num_blocks, block_size, num_kv_heads, head_size]
109 # to K_cache[num_blocks, num_kv_heads, head_size/8, block_size, 8]
104 - k_cache = k_cache.view(-1, block_size, num_heads, head_size // 8,
110 + k_cache = k_cache.view(-1, block_size, num_kv_heads, head_size // 8,
111 8).permute(0, 2, 3, 1, 4).contiguous()
112 # transpose V_cache[num_blocks, block_size, num_kv_heads, head_size]
113 # to V_cache[num_blocks, num_kv_heads, head_size, block_size]
108 - v_cache = v_cache.view(-1, block_size, num_heads,
114 + v_cache = v_cache.view(-1, block_size, num_kv_heads,
115 head_size).permute(0, 2, 3, 1).contiguous()
116
117 # Warm up the Triton kernel by calling it once before actually measuring generation time
@@ -123,12 +129,29 @@ def test_contexted_kv_attention(
129
130 attn_op = xops.fmha.cutlass.FwOp()
131
132 + if num_kv_heads != num_heads:
133 + # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
134 + # project the key and value tensors to the desired number of
135 + # heads.
136 + #
137 + # see also: vllm/model_executor/layers/attention.py
138 + query = query.view(query.shape[0], num_kv_heads, num_queries_per_kv,
139 + query.shape[-1])
140 + key = key[:, :, None, :].expand(key.shape[0], num_kv_heads,
141 + num_queries_per_kv, key.shape[-1])
142 + value = value[:, :,
143 + None, :].expand(value.shape[0], num_kv_heads,
144 + num_queries_per_kv, value.shape[-1])
145 + query = query.unsqueeze(0)
146 + key = key.unsqueeze(0)
147 + value = value.unsqueeze(0)
148 +
149 attn_bias = BlockDiagonalCausalFromBottomRightMask.from_seqlens(
150 subquery_lens, seq_lens)
151 output_ref = xops.memory_efficient_attention_forward(
129 - query.unsqueeze(0),
130 - key.unsqueeze(0),
131 - value.unsqueeze(0),
152 + query,
153 + key,
154 + value,
155 attn_bias=attn_bias,
156 p=0.0,
157 scale=scale,
@@ -137,9 +160,9 @@ def test_contexted_kv_attention(
160 torch.cuda.synchronize()
161 start_time = time.time()
162 output_ref = xops.memory_efficient_attention_forward(
140 - query.unsqueeze(0),
141 - key.unsqueeze(0),
142 - value.unsqueeze(0),
163 + query,
164 + key,
165 + value,
166 attn_bias=attn_bias,
167 p=0.0,
168 scale=scale,
@@ -148,5 +171,5 @@ def test_contexted_kv_attention(
171 torch.cuda.synchronize()
172 end_time = time.time()
173 print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
151 - output_ref = output_ref.squeeze(0)
174 + output_ref = output_ref.squeeze(0, 2)
175 assert torch.allclose(output_ref, output, atol=1e-6, rtol=0)
vllm/model_executor/layers/attention.py
+18 -16
@@ -137,25 +137,27 @@ class PagedAttention(nn.Module):
137 )
138
139 if input_metadata.is_prompt:
140 - # Prompt run.
141 - if self.num_kv_heads != self.num_heads:
142 - # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
143 - # project the key and value tensors to the desired number of
144 - # heads.
145 - # TODO(woosuk): Use MQA/GQA kernels for higher performance.
146 - query = query.view(query.shape[0], self.num_kv_heads,
147 - self.num_queries_per_kv, query.shape[-1])
148 - key = key[:, :,
149 - None, :].expand(key.shape[0], self.num_kv_heads,
150 - self.num_queries_per_kv,
151 - key.shape[-1])
152 - value = value[:, :, None, :].expand(value.shape[0],
153 - self.num_kv_heads,
154 - self.num_queries_per_kv,
155 - value.shape[-1])
140 # normal attention
141 if (key_cache is None or value_cache is None
142 or input_metadata.block_tables.numel() == 0):
143 + if self.num_kv_heads != self.num_heads:
144 + # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
145 + # project the key and value tensors to the desired number of
146 + # heads.
147 + # TODO(woosuk): Use MQA/GQA kernels for higher performance.
148 + query = query.view(query.shape[0], self.num_kv_heads,
149 + self.num_queries_per_kv,
150 + query.shape[-1])
151 + key = key[:, :,
152 + None, :].expand(key.shape[0], self.num_kv_heads,
153 + self.num_queries_per_kv,
154 + key.shape[-1])
155 + value = value[:, :,
156 + None, :].expand(value.shape[0],
157 + self.num_kv_heads,
158 + self.num_queries_per_kv,
159 + value.shape[-1])
160 +
161 # Set attention bias if not provided. This typically happens at
162 # the very attention layer of every iteration.
163 # FIXME(woosuk): This is a hack.
vllm/model_executor/layers/triton_kernel/prefix_prefill.py
+27 -12
@@ -45,6 +45,7 @@ if triton.__version__ >= "2.1.0":
45 stride_v_cache_h,
46 stride_v_cache_d,
47 stride_v_cache_bl,
48 + num_queries_per_kv: int,
49 BLOCK_M: tl.constexpr,
50 BLOCK_DMODEL: tl.constexpr,
51 BLOCK_N: tl.constexpr,
@@ -53,6 +54,8 @@ if triton.__version__ >= "2.1.0":
54 cur_head = tl.program_id(1)
55 start_m = tl.program_id(2)
56
57 + cur_kv_head = cur_head // num_queries_per_kv
58 +
59 cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch)
60 cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
61 cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
@@ -85,13 +88,14 @@ if triton.__version__ >= "2.1.0":
88 mask=(start_n + offs_n) < cur_batch_ctx_len,
89 other=0)
90 off_k = (bn[None, :] * stride_k_cache_bs +
88 - cur_head * stride_k_cache_h +
91 + cur_kv_head * stride_k_cache_h +
92 (offs_d[:, None] // x) * stride_k_cache_d +
93 ((start_n + offs_n[None, :]) % block_size) *
94 stride_k_cache_bl +
95 (offs_d[:, None] % x) * stride_k_cache_x)
96 off_v = (
94 - bn[:, None] * stride_v_cache_bs + cur_head * stride_v_cache_h +
97 + bn[:, None] * stride_v_cache_bs +
98 + cur_kv_head * stride_v_cache_h +
99 offs_d[None, :] * stride_v_cache_d +
100 (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
101 k = tl.load(K_cache + off_k,
@@ -131,9 +135,9 @@ if triton.__version__ >= "2.1.0":
135 l_i = l_i_new
136 m_i = m_i_new
137
134 - off_k = (offs_n[None, :] * stride_kbs + cur_head * stride_kh +
138 + off_k = (offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh +
139 offs_d[:, None] * stride_kd)
136 - off_v = (offs_n[:, None] * stride_vbs + cur_head * stride_vh +
140 + off_v = (offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh +
141 offs_d[None, :] * stride_vd)
142 k_ptrs = K + off_k
143 v_ptrs = V + off_v
@@ -232,6 +236,7 @@ if triton.__version__ >= "2.1.0":
236 stride_v_cache_h,
237 stride_v_cache_d,
238 stride_v_cache_bl,
239 + num_queries_per_kv: int,
240 BLOCK_M: tl.constexpr,
241 BLOCK_DMODEL: tl.constexpr,
242 BLOCK_N: tl.constexpr,
@@ -240,6 +245,8 @@ if triton.__version__ >= "2.1.0":
245 cur_head = tl.program_id(1)
246 start_m = tl.program_id(2)
247
248 + cur_kv_head = cur_head // num_queries_per_kv
249 +
250 cur_batch_ctx_len = tl.load(B_Ctxlen + cur_batch)
251 cur_batch_seq_len = tl.load(B_Seqlen + cur_batch)
252 cur_batch_in_all_start_index = tl.load(B_Start_Loc + cur_batch)
@@ -272,13 +279,14 @@ if triton.__version__ >= "2.1.0":
279 mask=(start_n + offs_n) < cur_batch_ctx_len,
280 other=0)
281 off_k = (bn[None, :] * stride_k_cache_bs +
275 - cur_head * stride_k_cache_h +
282 + cur_kv_head * stride_k_cache_h +
283 (offs_d[:, None] // x) * stride_k_cache_d +
284 ((start_n + offs_n[None, :]) % block_size) *
285 stride_k_cache_bl +
286 (offs_d[:, None] % x) * stride_k_cache_x)
287 off_v = (
281 - bn[:, None] * stride_v_cache_bs + cur_head * stride_v_cache_h +
288 + bn[:, None] * stride_v_cache_bs +
289 + cur_kv_head * stride_v_cache_h +
290 offs_d[None, :] * stride_v_cache_d +
291 (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
292 k = tl.load(K_cache + off_k,
@@ -317,9 +325,9 @@ if triton.__version__ >= "2.1.0":
325 l_i = l_i_new
326 m_i = m_i_new
327
320 - off_k = (offs_n[None, :] * stride_kbs + cur_head * stride_kh +
328 + off_k = (offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh +
329 offs_d[:, None] * stride_kd)
322 - off_v = (offs_n[:, None] * stride_vbs + cur_head * stride_vh +
330 + off_v = (offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh +
331 offs_d[None, :] * stride_vd)
332 k_ptrs = K + off_k
333 v_ptrs = V + off_v
@@ -420,6 +428,7 @@ if triton.__version__ >= "2.1.0":
428 stride_v_cache_h,
429 stride_v_cache_d,
430 stride_v_cache_bl,
431 + num_queries_per_kv: int,
432 BLOCK_M: tl.constexpr,
433 BLOCK_DMODEL: tl.constexpr,
434 BLOCK_N: tl.constexpr,
@@ -429,6 +438,8 @@ if triton.__version__ >= "2.1.0":
438 cur_head = tl.program_id(1)
439 start_m = tl.program_id(2)
440
441 + cur_kv_head = cur_head // num_queries_per_kv
442 +
443 # cur_batch_seq_len: the length of prompts
444 # cur_batch_ctx_len: the length of prefix
445 # cur_batch_in_all_start_index: the start id of the dim=0
@@ -468,13 +479,14 @@ if triton.__version__ >= "2.1.0":
479 mask=(start_n + offs_n) < cur_batch_ctx_len,
480 other=0)
481 off_k = (bn[None, :] * stride_k_cache_bs +
471 - cur_head * stride_k_cache_h +
482 + cur_kv_head * stride_k_cache_h +
483 (offs_d[:, None] // x) * stride_k_cache_d +
484 ((start_n + offs_n[None, :]) % block_size) *
485 stride_k_cache_bl +
486 (offs_d[:, None] % x) * stride_k_cache_x)
487 off_v = (
477 - bn[:, None] * stride_v_cache_bs + cur_head * stride_v_cache_h +
488 + bn[:, None] * stride_v_cache_bs +
489 + cur_kv_head * stride_v_cache_h +
490 offs_d[None, :] * stride_v_cache_d +
491 (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
492 k = tl.load(K_cache + off_k,
@@ -522,9 +534,9 @@ if triton.__version__ >= "2.1.0":
534 l_i = l_i_new
535 m_i = m_i_new
536
525 - off_k = (offs_n[None, :] * stride_kbs + cur_head * stride_kh +
537 + off_k = (offs_n[None, :] * stride_kbs + cur_kv_head * stride_kh +
538 offs_d[:, None] * stride_kd)
527 - off_v = (offs_n[:, None] * stride_vbs + cur_head * stride_vh +
539 + off_v = (offs_n[:, None] * stride_vbs + cur_kv_head * stride_vh +
540 offs_d[None, :] * stride_vd)
541 k_ptrs = K + off_k
542 v_ptrs = V + off_v
@@ -628,6 +640,7 @@ if triton.__version__ >= "2.1.0":
640
641 sm_scale = 1.0 / (Lq**0.5)
642 batch, head = b_seq_len.shape[0], q.shape[1]
643 + num_queries_per_kv = q.shape[1] // k.shape[1]
644
645 grid = (batch, head, triton.cdiv(max_input_len, BLOCK)) # batch, head,
646
@@ -674,6 +687,7 @@ if triton.__version__ >= "2.1.0":
687 v_cache.stride(2),
688 v_cache.stride(
689 3), #[num_blocks, num_kv_heads, head_size, block_size]
690 + num_queries_per_kv=num_queries_per_kv,
691 BLOCK_M=BLOCK,
692 BLOCK_DMODEL=Lk,
693 BLOCK_N=BLOCK,
@@ -721,6 +735,7 @@ if triton.__version__ >= "2.1.0":
735 v_cache.stride(2),
736 v_cache.stride(
737 3), #[num_blocks, num_kv_heads, head_size, block_size]
738 + num_queries_per_kv=num_queries_per_kv,
739 BLOCK_M=BLOCK,
740 BLOCK_DMODEL=Lk,
741 BLOCK_N=BLOCK,