@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0f9a6e3d22
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix][Kernel] allow non-power-of-2 for prefix prefill with alibi (#4573)

DefTruth committed May 9, 2024 at 00:19 UTC 0f9a6e3d229cade0ae9a53a4f69a38f52e430bd0
2 files changed +267 -17
tests/kernels/test_prefix_prefill.py
+242 -1
@@ -1,3 +1,4 @@
1 +import math
2 import random
3 import time
4
@@ -6,11 +7,12 @@ import torch
7 from xformers import ops as xops
8 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
9
10 +from vllm.attention.backends.xformers import _make_alibi_bias
11 from vllm.attention.ops.prefix_prefill import context_attention_fwd
12
13 NUM_HEADS = [64]
14 NUM_QUERIES_PER_KV = [1, 8, 64]
13 -HEAD_SIZES = [128, 96]
15 +HEAD_SIZES = [128, 96, 24]
16 DTYPES = [torch.float16]
17 CUDA_DEVICES = [
18 f"cuda:{i}" for i in range(1 if torch.cuda.device_count() == 1 else 2)
@@ -207,3 +209,242 @@ def test_contexted_kv_attention(
209 print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
210 output_ref = output_ref.reshape(output.shape)
211 assert torch.allclose(output_ref, output, atol=1e-6, rtol=0)
212 +
213 +
214 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
215 +@pytest.mark.parametrize("num_queries_per_kv", NUM_QUERIES_PER_KV)
216 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
217 +@pytest.mark.parametrize("dtype", DTYPES)
218 +@pytest.mark.parametrize("device", CUDA_DEVICES)
219 +@torch.inference_mode()
220 +def test_contexted_kv_attention_alibi(
221 + num_heads: int,
222 + num_queries_per_kv: int,
223 + head_size: int,
224 + dtype: torch.dtype,
225 + device: str,
226 +) -> None:
227 + random.seed(0)
228 + torch.manual_seed(0)
229 + if torch.cuda.is_available():
230 + torch.cuda.manual_seed(0)
231 + torch.set_default_device(device)
232 +
233 + # Need this, otherwise when we capture the graph the process
234 + # for GPU 1 would run on both GPU0 and GPU1 and things would hang
235 + #
236 + # see also similar issue: https://github.com/Dao-AILab/flash-attention/issues/523
237 + torch.cuda.set_device(device)
238 +
239 + def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
240 + # Fork from: vllm/vllm/model_executor/models/bloom.py#L44
241 + closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
242 + base = torch.tensor(
243 + 2**(-(2**-(math.log2(closest_power_of_2) - 3))),
244 + dtype=torch.float32,
245 + )
246 + powers = torch.arange(1, 1 + closest_power_of_2, dtype=torch.int32)
247 + slopes = torch.pow(base, powers)
248 +
249 + if closest_power_of_2 != total_num_heads:
250 + extra_base = torch.tensor(
251 + 2**(-(2**-(math.log2(2 * closest_power_of_2) - 3))),
252 + dtype=torch.float32,
253 + )
254 + num_remaining_heads = min(closest_power_of_2,
255 + total_num_heads - closest_power_of_2)
256 + extra_powers = torch.arange(start=1,
257 + end=1 + 2 * num_remaining_heads,
258 + step=2,
259 + dtype=torch.int32)
260 + slopes = torch.cat(
261 + [slopes, torch.pow(extra_base, extra_powers)], dim=0)
262 + return slopes
263 +
264 + alibi_slopes = _get_alibi_slopes(num_heads).to(device)
265 +
266 + MAX_SEQ_LEN = 1024
267 + MAX_CTX_LEN = 1024
268 + BS = 10
269 + cache_size = 640
270 + block_size = 32
271 + max_block_per_request = 64
272 + query_lens = [random.randint(16, MAX_SEQ_LEN) for _ in range(BS)]
273 + ctx_lens = [random.randint(16, MAX_CTX_LEN) for _ in range(BS)]
274 + seq_lens = [a + b for a, b in zip(query_lens, ctx_lens)]
275 + num_kv_heads = num_heads // num_queries_per_kv
276 +
277 + num_tokens = sum(query_lens)
278 + query = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
279 + query.uniform_(-1e-3, 1e-3)
280 + output = torch.empty(num_tokens, num_heads, head_size, dtype=dtype)
281 +
282 + kv = torch.empty(sum(seq_lens), 2, num_kv_heads, head_size, dtype=dtype)
283 + kv.uniform_(-1e-3, 1e-3)
284 + key, value = kv.unbind(dim=1)
285 +
286 + k_cache = torch.zeros(cache_size,
287 + block_size,
288 + num_kv_heads,
289 + head_size,
290 + dtype=dtype)
291 + v_cache = torch.zeros(cache_size,
292 + block_size,
293 + num_kv_heads,
294 + head_size,
295 + dtype=dtype)
296 + k = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
297 + v = torch.zeros(sum(query_lens), num_kv_heads, head_size, dtype=dtype)
298 + values = torch.arange(0, cache_size, dtype=torch.long)
299 + values = values[torch.randperm(cache_size)]
300 + block_table = values[:BS * max_block_per_request].view(
301 + BS, max_block_per_request)
302 + b_seq_len = torch.tensor(seq_lens, dtype=torch.long)
303 + b_ctx_len = torch.tensor(ctx_lens, dtype=torch.long)
304 + b_start_loc = torch.cumsum(torch.tensor([0] + query_lens[:-1],
305 + dtype=torch.long),
306 + dim=0)
307 + max_input_len = MAX_SEQ_LEN
308 + # copy kv to cache
309 + b_seq_start_loc = torch.cumsum(torch.tensor([0] + seq_lens[:-1],
310 + dtype=torch.long),
311 + dim=0)
312 + for i in range(BS):
313 + for j in range(query_lens[i]):
314 + k[b_start_loc[i] + j].copy_(key[b_seq_start_loc[i] + b_ctx_len[i] +
315 + j])
316 + v[b_start_loc[i] + j].copy_(value[b_seq_start_loc[i] +
317 + b_ctx_len[i] + j])
318 + cur_ctx = 0
319 + block_id = 0
320 + while cur_ctx < b_ctx_len[i]:
321 + start_loc = b_seq_start_loc[i] + cur_ctx
322 + if cur_ctx + block_size > b_ctx_len[i]:
323 + end_loc = b_seq_start_loc[i] + b_ctx_len[i]
324 + else:
325 + end_loc = start_loc + block_size
326 + start_slot = block_table[i, block_id] * block_size
327 + end_slot = start_slot + end_loc - start_loc
328 + k_cache.view(-1, num_kv_heads,
329 + head_size)[start_slot:end_slot].copy_(
330 + key[start_loc:end_loc])
331 + v_cache.view(-1, num_kv_heads,
332 + head_size)[start_slot:end_slot].copy_(
333 + value[start_loc:end_loc])
334 + cur_ctx += block_size
335 + block_id += 1
336 + # transpose K_cache[num_blocks, block_size, num_kv_heads, head_size]
337 + # to K_cache[num_blocks, num_kv_heads, head_size/8, block_size, 8]
338 + k_cache = k_cache.view(-1, block_size, num_kv_heads, head_size // 8,
339 + 8).permute(0, 2, 3, 1, 4).contiguous()
340 + # transpose V_cache[num_blocks, block_size, num_kv_heads, head_size]
341 + # to V_cache[num_blocks, num_kv_heads, head_size, block_size]
342 + v_cache = v_cache.view(-1, block_size, num_kv_heads,
343 + head_size).permute(0, 2, 3, 1).contiguous()
344 +
345 + # Warm up the Triton kernel by calling it once before actually measuring
346 + # generation time
347 + context_attention_fwd(query,
348 + k,
349 + v,
350 + output,
351 + k_cache,
352 + v_cache,
353 + block_table,
354 + b_start_loc,
355 + b_seq_len,
356 + b_ctx_len,
357 + max_input_len,
358 + alibi_slopes=alibi_slopes)
359 + torch.cuda.synchronize()
360 + start_time = time.time()
361 + context_attention_fwd(query,
362 + k,
363 + v,
364 + output,
365 + k_cache,
366 + v_cache,
367 + block_table,
368 + b_start_loc,
369 + b_seq_len,
370 + b_ctx_len,
371 + max_input_len,
372 + alibi_slopes=alibi_slopes)
373 + torch.cuda.synchronize()
374 + end_time = time.time()
375 + print(f"triton Time: {(end_time - start_time)*1000:.2f} ms")
376 + scale = float(1.0 / (head_size**0.5))
377 +
378 + # NOTE(DefTruth): In order to reuse _make_alibi_bias function,
379 + # we have to pad query tensor before MQA/GQA expanding.
380 + if query.shape[0] != key.shape[0]:
381 + query_pad = torch.empty(sum(seq_lens),
382 + num_heads,
383 + head_size,
384 + dtype=dtype)
385 + query_pad.uniform_(-1e-3, 1e-3)
386 + seq_start = 0
387 + query_start = 0
388 + for i, (query_len, seq_len) in enumerate(zip(query_lens, seq_lens)):
389 + seq_end = seq_start + seq_len
390 + query_end = query_start + query_len
391 + query_pad[seq_start:seq_end, ...] = torch.cat([
392 + torch.zeros(
393 + seq_len - query_len, num_heads, head_size, dtype=dtype),
394 + query[query_start:query_end, ...]
395 + ],
396 + dim=0)
397 + seq_start += seq_len
398 + query_start += query_len
399 + query = query_pad
400 +
401 + if num_kv_heads != num_heads:
402 + # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
403 + # project the key and value tensors to the desired number of
404 + # heads.
405 + #
406 + # see also: vllm/model_executor/layers/attention.py
407 + query = query.view(query.shape[0], num_kv_heads, num_queries_per_kv,
408 + query.shape[-1])
409 + key = key[:, :, None, :].expand(key.shape[0], num_kv_heads,
410 + num_queries_per_kv, key.shape[-1])
411 + value = value[:, :,
412 + None, :].expand(value.shape[0], num_kv_heads,
413 + num_queries_per_kv, value.shape[-1])
414 +
415 + query = query.unsqueeze(0)
416 + key = key.unsqueeze(0)
417 + value = value.unsqueeze(0)
418 +
419 + attn_bias = _make_alibi_bias(alibi_slopes, num_kv_heads, dtype, seq_lens)
420 + output_ref = torch.empty_like(output)
421 + seq_start = 0
422 + query_start = 0
423 + start_time = time.time()
424 + # Attention with alibi slopes.
425 + # FIXME(DefTruth): Because xformers does not support dynamic sequence
426 + # lengths with custom attention bias, we process each prompt one by
427 + # one. This is inefficient, especially when we have many short prompts.
428 + # modified from: vllm/attention/backends/xformers.py#L343
429 + for i, (query_len, seq_len) in enumerate(zip(query_lens, seq_lens)):
430 + seq_end = seq_start + seq_len
431 + query_end = query_start + query_len
432 + out = xops.memory_efficient_attention_forward(query[:,
433 + seq_start:seq_end],
434 + key[:,
435 + seq_start:seq_end],
436 + value[:,
437 + seq_start:seq_end],
438 + attn_bias=attn_bias[i],
439 + p=0.0,
440 + scale=scale)
441 + out = out.view_as(query[:, seq_start:seq_end]).view(
442 + seq_len, num_heads, head_size)
443 + output_ref[query_start:query_end, ...].copy_(out[seq_len - query_len:,
444 + ...])
445 + seq_start += seq_len
446 + query_start += query_len
447 + torch.cuda.synchronize()
448 + end_time = time.time()
449 + print(f"xformers Time: {(end_time - start_time)*1000:.2f} ms")
450 + assert torch.allclose(output_ref, output, atol=1e-6, rtol=0)
vllm/attention/ops/prefix_prefill.py
+25 -16
@@ -472,7 +472,8 @@ if triton.__version__ >= "2.1.0":
472 stride_v_cache_bl,
473 num_queries_per_kv: int,
474 BLOCK_M: tl.constexpr,
475 - BLOCK_DMODEL: tl.constexpr,
475 + BLOCK_DMODEL: tl.constexpr, # head size
476 + BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2
477 BLOCK_N: tl.constexpr,
478 ):
479 # attn_bias[]
@@ -493,21 +494,24 @@ if triton.__version__ >= "2.1.0":
494
495 # initialize offsets
496 offs_n = tl.arange(0, BLOCK_N)
496 - offs_d = tl.arange(0, BLOCK_DMODEL)
497 + offs_d = tl.arange(0, BLOCK_DMODEL_PADDED)
498 offs_m = start_m * BLOCK_M + tl.arange(0, BLOCK_M)
499 off_q = (
500 (cur_batch_in_all_start_index + offs_m[:, None]) * stride_qbs +
501 cur_head * stride_qh + offs_d[None, :] * stride_qd)
502
502 - q = tl.load(
503 - Q + off_q,
504 - mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len,
505 - other=0.0)
503 + dim_mask = tl.where(
504 + tl.arange(0, BLOCK_DMODEL_PADDED) < BLOCK_DMODEL, 1, 0).to(tl.int1)
505 +
506 + q = tl.load(Q + off_q,
507 + mask=dim_mask[None, :] &
508 + (offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len),
509 + other=0.0)
510
511 # # initialize pointer to m and l
512 m_i = tl.zeros([BLOCK_M], dtype=tl.float32) - float("inf")
513 l_i = tl.zeros([BLOCK_M], dtype=tl.float32)
510 - acc = tl.zeros([BLOCK_M, BLOCK_DMODEL], dtype=tl.float32)
514 + acc = tl.zeros([BLOCK_M, BLOCK_DMODEL_PADDED], dtype=tl.float32)
515
516 alibi_slope = tl.load(Alibi_slopes + cur_head)
517 alibi_start_q = tl.arange(
@@ -532,8 +536,9 @@ if triton.__version__ >= "2.1.0":
536 offs_d[None, :] * stride_v_cache_d +
537 (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl)
538 k = tl.load(K_cache + off_k,
535 - mask=(start_n + offs_n[None, :]) < cur_batch_ctx_len,
536 - other=0.0)
539 + mask=dim_mask[:, None] &
540 + ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
541 + other=0.0) # [D,N]
542
543 qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
544 qk += tl.dot(q, k)
@@ -567,7 +572,8 @@ if triton.__version__ >= "2.1.0":
572 acc = acc * acc_scale[:, None]
573 # update acc
574 v = tl.load(V_cache + off_v,
570 - mask=(start_n + offs_n[:, None]) < cur_batch_ctx_len,
575 + mask=dim_mask[None, :] &
576 + ((start_n + offs_n[:, None]) < cur_batch_ctx_len),
577 other=0.0)
578
579 p = p.to(v.dtype)
@@ -600,8 +606,9 @@ if triton.__version__ >= "2.1.0":
606 # -- compute qk ----
607 k = tl.load(k_ptrs +
608 (cur_batch_in_all_start_index + start_n) * stride_kbs,
603 - mask=(start_n + offs_n[None, :]) <
604 - cur_batch_seq_len - cur_batch_ctx_len,
609 + mask=dim_mask[:, None] &
610 + ((start_n + offs_n[None, :]) <
611 + cur_batch_seq_len - cur_batch_ctx_len),
612 other=0.0)
613
614 qk = tl.zeros([BLOCK_M, BLOCK_N], dtype=tl.float32)
@@ -637,8 +644,9 @@ if triton.__version__ >= "2.1.0":
644 # update acc
645 v = tl.load(v_ptrs +
646 (cur_batch_in_all_start_index + start_n) * stride_vbs,
640 - mask=(start_n + offs_n[:, None]) <
641 - cur_batch_seq_len - cur_batch_ctx_len,
647 + mask=dim_mask[None, :] &
648 + ((start_n + offs_n[:, None]) <
649 + cur_batch_seq_len - cur_batch_ctx_len),
650 other=0.0)
651
652 p = p.to(v.dtype)
@@ -656,7 +664,8 @@ if triton.__version__ >= "2.1.0":
664 out_ptrs = Out + off_o
665 tl.store(out_ptrs,
666 acc,
659 - mask=offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len)
667 + mask=dim_mask[None, :] &
668 + (offs_m[:, None] < cur_batch_seq_len - cur_batch_ctx_len))
669 return
670
671 @torch.inference_mode()
@@ -690,7 +699,6 @@ if triton.__version__ >= "2.1.0":
699
700 num_warps = 8 if Lk <= 64 else 8
701 if alibi_slopes is not None:
693 - assert Lk == Lk_padded
702 _fwd_kernel_alibi[grid](
703 q,
704 k,
@@ -735,6 +743,7 @@ if triton.__version__ >= "2.1.0":
743 num_queries_per_kv=num_queries_per_kv,
744 BLOCK_M=BLOCK,
745 BLOCK_DMODEL=Lk,
746 + BLOCK_DMODEL_PADDED=Lk_padded,
747 BLOCK_N=BLOCK,
748 num_warps=num_warps,
749 num_stages=1,