@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 75e94309e8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Perf] Mem align KV caches for CUDA devices (MLA perf improvement) (#12676)

Signed-off-by: simon-mo <xmo@berkeley.edu> Signed-off-by: Lucas Wilkinson <lcwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: simon-mo <xmo@berkeley.edu>

Lucas Wilkinson committed Feb 4, 2025 at 21:22 UTC 75e94309e8d8919e0daea041f6cd81a4b8c09060
10 files changed +429 -34
csrc/cache.h
+3
@@ -15,6 +15,9 @@ void copy_blocks(std::vector<torch::Tensor> const& key_caches,
15 std::vector<torch::Tensor> const& value_caches,
16 const torch::Tensor& block_mapping);
17
18 +void copy_blocks_mla(std::vector<torch::Tensor> const& kv_caches,
19 + const torch::Tensor& block_mapping);
20 +
21 void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
22 torch::Tensor& key_cache, torch::Tensor& value_cache,
23 torch::Tensor& slot_mapping,
csrc/cache_kernels.cu
+70 -12
@@ -46,7 +46,10 @@ void swap_blocks(torch::Tensor& src, torch::Tensor& dst,
46 char* src_ptr = static_cast<char*>(src.data_ptr());
47 char* dst_ptr = static_cast<char*>(dst.data_ptr());
48
49 - const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
49 + // We use the stride instead of numel in case the cache is padded for memory
50 + // alignment reasons, we assume the blocks data (inclusive of any padding)
51 + // is contiguous in memory
52 + const int64_t block_size_in_bytes = src.element_size() * src.stride(0);
53 const at::cuda::OptionalCUDAGuard device_guard(
54 src_device.is_cuda() ? src_device : dst_device);
55 const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
@@ -93,6 +96,24 @@ __global__ void copy_blocks_kernel(int64_t* key_cache_ptrs,
96 }
97 }
98
99 +// Kernel for MLA, which works on a single joint kv_cache
100 +// Grid: (num_layers, num_pairs)
101 +template <typename scalar_t>
102 +__global__ void copy_blocks_mla_kernel(
103 + int64_t* cache_ptrs, const int64_t* __restrict__ block_mapping,
104 + const int mem_footprint_per_block) {
105 + const int layer_idx = blockIdx.x;
106 + const int pair_idx = blockIdx.y;
107 + scalar_t* cache = reinterpret_cast<scalar_t*>(cache_ptrs[layer_idx]);
108 + int64_t src_block = block_mapping[2 * pair_idx];
109 + int64_t dst_block = block_mapping[2 * pair_idx + 1];
110 + int64_t src_offset = src_block * mem_footprint_per_block;
111 + int64_t dst_offset = dst_block * mem_footprint_per_block;
112 + for (int i = threadIdx.x; i < mem_footprint_per_block; i += blockDim.x) {
113 + cache[dst_offset + i] = cache[src_offset + i];
114 + }
115 +}
116 +
117 } // namespace vllm
118
119 // Note: the key_caches and value_caches vectors are constant but
@@ -147,6 +168,42 @@ void copy_blocks(std::vector<torch::Tensor> const& key_caches,
168 }));
169 }
170
171 +// copy blocks kernel for MLA (assumes a joint KV-cache)
172 +void copy_blocks_mla(std::vector<torch::Tensor> const& kv_caches,
173 + const torch::Tensor& block_mapping) {
174 + int num_layers = kv_caches.size();
175 + if (num_layers == 0) {
176 + return;
177 + }
178 + torch::Device cache_device = kv_caches[0].device();
179 + TORCH_CHECK(cache_device.is_cuda(), "kv_cache must be on CUDA");
180 +
181 + std::vector<int64_t> cache_ptrs(num_layers);
182 + for (int layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
183 + cache_ptrs[layer_idx] =
184 + reinterpret_cast<int64_t>(kv_caches[layer_idx].data_ptr());
185 + }
186 + torch::Tensor cache_ptrs_tensor =
187 + torch::from_blob(cache_ptrs.data(), {num_layers}, torch::kInt64)
188 + .to(cache_device);
189 +
190 + int num_pairs = block_mapping.size(0);
191 + // We use the stride instead of numel in case the cache is padded for memory
192 + // alignment reasons, we assume the blocks data (inclusive of any padding)
193 + // is contiguous in memory
194 + int mem_footprint_per_block = kv_caches[0].stride(0);
195 + dim3 grid(num_layers, num_pairs);
196 + dim3 block(std::min(1024, mem_footprint_per_block));
197 + const at::cuda::OptionalCUDAGuard device_guard(cache_device);
198 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
199 + VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(
200 + kv_caches[0].scalar_type(), "copy_blocks_mla_kernel", ([&] {
201 + vllm::copy_blocks_mla_kernel<scalar_t><<<grid, block, 0, stream>>>(
202 + cache_ptrs_tensor.data_ptr<int64_t>(),
203 + block_mapping.data_ptr<int64_t>(), mem_footprint_per_block);
204 + }));
205 +}
206 +
207 namespace vllm {
208
209 template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
@@ -254,6 +311,7 @@ __global__ void concat_and_cache_mla_kernel(
311 // + pe_dim)]
312 const int64_t* __restrict__ slot_mapping, // [num_tokens]
313 const int block_stride, //
314 + const int entry_stride, //
315 const int kv_c_stride, //
316 const int k_pe_stride, //
317 const int kv_lora_rank, //
@@ -274,9 +332,8 @@ __global__ void concat_and_cache_mla_kernel(
332 int src_stride, int dst_stride, int size, int offset) {
333 for (int i = threadIdx.x; i < size; i += blockDim.x) {
334 const int64_t src_idx = token_idx * src_stride + i;
277 - const int64_t dst_idx = block_idx * block_stride +
278 - block_offset * (kv_lora_rank + pe_dim) + i +
279 - offset;
335 + const int64_t dst_idx =
336 + block_idx * block_stride + block_offset * entry_stride + i + offset;
337 if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
338 dst[dst_idx] = src[src_idx];
339 } else {
@@ -391,14 +448,14 @@ void reshape_and_cache_flash(
448 // KV_T is the stored data type of kv-cache.
449 // CACHE_T is the data type of key and value tensors.
450 // KV_DTYPE is the real data type of kv-cache.
394 -#define CALL_CONCAT_AND_CACHE_MLA(KV_T, CACHE_T, KV_DTYPE) \
395 - vllm::concat_and_cache_mla_kernel<KV_T, CACHE_T, KV_DTYPE> \
396 - <<<grid, block, 0, stream>>>( \
397 - reinterpret_cast<KV_T*>(kv_c.data_ptr()), \
398 - reinterpret_cast<KV_T*>(k_pe.data_ptr()), \
399 - reinterpret_cast<CACHE_T*>(kv_cache.data_ptr()), \
400 - slot_mapping.data_ptr<int64_t>(), block_stride, kv_c_stride, \
401 - k_pe_stride, kv_lora_rank, pe_dim, block_size, \
451 +#define CALL_CONCAT_AND_CACHE_MLA(KV_T, CACHE_T, KV_DTYPE) \
452 + vllm::concat_and_cache_mla_kernel<KV_T, CACHE_T, KV_DTYPE> \
453 + <<<grid, block, 0, stream>>>( \
454 + reinterpret_cast<KV_T*>(kv_c.data_ptr()), \
455 + reinterpret_cast<KV_T*>(k_pe.data_ptr()), \
456 + reinterpret_cast<CACHE_T*>(kv_cache.data_ptr()), \
457 + slot_mapping.data_ptr<int64_t>(), block_stride, entry_stride, \
458 + kv_c_stride, k_pe_stride, kv_lora_rank, pe_dim, block_size, \
459 reinterpret_cast<const float*>(scale.data_ptr()));
460
461 void concat_and_cache_mla(
@@ -428,6 +485,7 @@ void concat_and_cache_mla(
485 int kv_c_stride = kv_c.stride(0);
486 int k_pe_stride = k_pe.stride(0);
487 int block_stride = kv_cache.stride(0);
488 + int entry_stride = kv_cache.stride(1);
489
490 dim3 grid(num_tokens);
491 dim3 block(std::min(kv_lora_rank, 512));
csrc/torch_bindings.cpp
+4
@@ -450,6 +450,10 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
450 "Tensor block_mapping) -> ()");
451 cache_ops.impl("copy_blocks", torch::kCUDA, &copy_blocks);
452
453 + cache_ops.def(
454 + "copy_blocks_mla(Tensor(a!)[] kv_caches, Tensor block_mapping) -> ()");
455 + cache_ops.impl("copy_blocks_mla", torch::kCUDA, &copy_blocks_mla);
456 +
457 // Reshape the key and value tensors and cache them.
458 cache_ops.def(
459 "reshape_and_cache(Tensor key, Tensor value,"
tests/kernels/test_cache.py
+262
@@ -9,6 +9,7 @@ import torch
9 from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
10 from vllm import _custom_ops as ops
11 from vllm.platforms import current_platform
12 +from vllm.utils import align_to_256bytes
13
14 COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
15 DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -18,6 +19,13 @@ NUM_HEADS = [8] # Arbitrary values for testing
19 HEAD_SIZES = [64, 80, 120, 256]
20 BLOCK_SIZES = [8, 16, 32]
21
22 +# Parameters for MLA tests.
23 +KV_LORA_RANKS = [512]
24 +QK_ROPE_HEAD_DIMS = [64]
25 +NUM_TOKENS_MLA = [42]
26 +BLOCK_SIZES_MLA = [16]
27 +NUM_BLOCKS_MLA = [8]
28 +
29 # Arbitrary values for testing
30 # don't make it too large. e.g. [1024, 36000] will OOM
31 NUM_BLOCKS = [1024, 10000]
@@ -432,3 +440,257 @@ def test_fp8_e4m3_conversion(
440 ops.convert_fp8(converted_cache, cache_fp8)
441
442 torch.testing.assert_close(cache, converted_cache, atol=0.001, rtol=0.1)
443 +
444 +
445 +def _create_mla_cache(
446 + num_blocks: int,
447 + block_size: int,
448 + entry_size: int,
449 + dtype: torch.dtype,
450 + kv_cache_dtype: str,
451 + device: str,
452 + align_cache: bool,
453 +) -> torch.Tensor:
454 + cache_dtype = torch.uint8 if kv_cache_dtype == "fp8" else dtype
455 +
456 + if align_cache:
457 + alloc_entry_size = align_to_256bytes(entry_size, cache_dtype)
458 + alloc_shape = (num_blocks, block_size, alloc_entry_size)
459 + cache_full = torch.zeros(alloc_shape, dtype=cache_dtype, device=device)
460 + cache = cache_full[..., :entry_size]
461 + else:
462 + cache = torch.zeros(num_blocks,
463 + block_size,
464 + entry_size,
465 + dtype=cache_dtype,
466 + device=device)
467 + return cache
468 +
469 +
470 +def _fill_mla_cache(cache: torch.Tensor, kv_cache_dtype: str):
471 + rand_dtype = torch.float16 if kv_cache_dtype == "fp8" else cache.dtype
472 +
473 + vals = torch.randn(*cache.shape, device=cache.device, dtype=rand_dtype)
474 + if kv_cache_dtype == "fp8":
475 + temp = torch.zeros_like(cache)
476 + ops.convert_fp8(temp, vals, 1.0, kv_dtype=kv_cache_dtype)
477 + vals = temp
478 + cache.copy_(vals)
479 +
480 +
481 +@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
482 +@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
483 +@pytest.mark.parametrize("num_tokens", NUM_TOKENS_MLA)
484 +@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
485 +@pytest.mark.parametrize("num_blocks", NUM_BLOCKS_MLA)
486 +@pytest.mark.parametrize("dtype", DTYPES)
487 +@pytest.mark.parametrize("seed", SEEDS)
488 +@pytest.mark.parametrize("device", CUDA_DEVICES)
489 +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
490 +@pytest.mark.parametrize("align_cache", [False])
491 +@torch.inference_mode()
492 +def test_concat_and_cache_mla(
493 + kv_lora_rank: int,
494 + qk_rope_head_dim: int,
495 + num_tokens: int,
496 + block_size: int,
497 + num_blocks: int,
498 + dtype: torch.dtype,
499 + seed: int,
500 + device: str,
501 + kv_cache_dtype: str,
502 + align_cache: bool,
503 +) -> None:
504 + current_platform.seed_everything(seed)
505 + torch.set_default_device(device)
506 +
507 + total_slots = num_blocks * block_size
508 + slot_mapping_lst = random.sample(range(total_slots), num_tokens)
509 + slot_mapping = torch.tensor(slot_mapping_lst,
510 + dtype=torch.long,
511 + device=device)
512 +
513 + kv_c = torch.randn(num_tokens, kv_lora_rank, dtype=dtype, device=device)
514 + k_pe = torch.randn(num_tokens,
515 + qk_rope_head_dim,
516 + dtype=dtype,
517 + device=device)
518 + entry_size = kv_lora_rank + qk_rope_head_dim
519 +
520 + scale = torch.tensor(0.1, dtype=torch.float32, device=device)
521 + kv_cache = _create_mla_cache(num_blocks, block_size, entry_size, dtype,
522 + kv_cache_dtype, device, align_cache)
523 + ref_temp = torch.zeros(*kv_cache.shape, dtype=dtype, device=device)
524 +
525 + for i in range(num_tokens):
526 + slot = slot_mapping[i].item()
527 + block_idx = slot // block_size
528 + block_offset = slot % block_size
529 + ref_temp[block_idx, block_offset, :kv_lora_rank] = kv_c[i]
530 + ref_temp[block_idx, block_offset, kv_lora_rank:] = k_pe[i]
531 +
532 + if kv_cache_dtype == "fp8":
533 + ref_kv_cache = torch.empty_like(ref_temp, dtype=kv_cache.dtype)
534 + ops.convert_fp8(ref_kv_cache,
535 + ref_temp,
536 + scale.item(),
537 + kv_dtype=kv_cache_dtype)
538 + else:
539 + ref_kv_cache = ref_temp
540 +
541 + opcheck(
542 + torch.ops._C_cache_ops.concat_and_cache_mla,
543 + (kv_c, k_pe, kv_cache, slot_mapping, kv_cache_dtype, scale),
544 + test_utils=DEFAULT_OPCHECK_TEST_UTILS,
545 + )
546 +
547 + ops.concat_and_cache_mla(kv_c, k_pe, kv_cache, slot_mapping,
548 + kv_cache_dtype, scale)
549 +
550 + if kv_cache_dtype == "fp8":
551 + result_temp = torch.empty_like(kv_cache, dtype=torch.float16)
552 + ops.convert_fp8(result_temp,
553 + kv_cache.contiguous(),
554 + scale.item(),
555 + kv_dtype=kv_cache_dtype)
556 + expected_temp = torch.empty_like(ref_kv_cache, dtype=torch.float16)
557 + ops.convert_fp8(expected_temp,
558 + ref_kv_cache,
559 + scale.item(),
560 + kv_dtype=kv_cache_dtype)
561 + torch.testing.assert_close(result_temp,
562 + expected_temp,
563 + atol=0.001,
564 + rtol=0.1)
565 + else:
566 + torch.testing.assert_close(kv_cache, ref_kv_cache)
567 +
568 +
569 +@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
570 +@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
571 +@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
572 +@pytest.mark.parametrize("num_blocks", NUM_BLOCKS_MLA)
573 +@pytest.mark.parametrize("num_layers", NUM_LAYERS)
574 +@pytest.mark.parametrize("dtype", DTYPES)
575 +@pytest.mark.parametrize("seed", SEEDS)
576 +@pytest.mark.parametrize("device", CUDA_DEVICES)
577 +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
578 +@pytest.mark.parametrize("align_cache", [False, True])
579 +@torch.inference_mode()
580 +def test_copy_blocks_mla(
581 + kv_lora_rank: int,
582 + qk_rope_head_dim: int,
583 + block_size: int,
584 + num_blocks: int,
585 + num_layers: int,
586 + dtype: torch.dtype,
587 + seed: int,
588 + device: str,
589 + kv_cache_dtype: str,
590 + align_cache: bool,
591 +) -> None:
592 + current_platform.seed_everything(seed)
593 + torch.set_default_device(device)
594 +
595 + entry_size = kv_lora_rank + qk_rope_head_dim
596 +
597 + kv_caches = []
598 + for _ in range(num_layers):
599 + kv_cache = _create_mla_cache(num_blocks, block_size, entry_size, dtype,
600 + kv_cache_dtype, device, align_cache)
601 + _fill_mla_cache(kv_cache, kv_cache_dtype=kv_cache_dtype)
602 + kv_caches.append(kv_cache)
603 +
604 + ref_caches = [kv_cache.clone() for kv_cache in kv_caches]
605 +
606 + num_mappings = min(2, num_blocks // 2)
607 + src_blocks = random.sample(range(num_blocks), num_mappings)
608 + remaining = list(set(range(num_blocks)) - set(src_blocks))
609 + dst_blocks = random.sample(remaining, 2 * num_mappings)
610 + block_mapping = []
611 + for i in range(num_mappings):
612 + src = src_blocks[i]
613 + dst1 = dst_blocks[2 * i]
614 + dst2 = dst_blocks[2 * i + 1]
615 + block_mapping.append((src, dst1))
616 + block_mapping.append((src, dst2))
617 + block_mapping_tensor = torch.tensor(block_mapping,
618 + dtype=torch.int64,
619 + device=device).view(-1, 2)
620 +
621 + for src, dst in block_mapping:
622 + for ref_cache in ref_caches:
623 + ref_cache[dst].copy_(ref_cache[src])
624 +
625 + opcheck(
626 + torch.ops._C_cache_ops.copy_blocks_mla,
627 + (kv_caches, block_mapping_tensor),
628 + test_utils=DEFAULT_OPCHECK_TEST_UTILS,
629 + )
630 + ops.copy_blocks_mla(kv_caches, block_mapping_tensor)
631 +
632 + for kv_cache, ref_cache in zip(kv_caches, ref_caches):
633 + torch.testing.assert_close(kv_cache, ref_cache)
634 +
635 +
636 +@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
637 +@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
638 +@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
639 +@pytest.mark.parametrize("num_blocks", NUM_BLOCKS_MLA)
640 +@pytest.mark.parametrize("dtype", DTYPES)
641 +@pytest.mark.parametrize("seed", SEEDS)
642 +@pytest.mark.parametrize("device", CUDA_DEVICES)
643 +@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
644 +@pytest.mark.parametrize("align_cache", [False, True])
645 +@torch.inference_mode()
646 +def test_swap_blocks_mla(
647 + kv_lora_rank: int,
648 + qk_rope_head_dim: int,
649 + block_size: int,
650 + num_blocks: int,
651 + dtype: torch.dtype,
652 + seed: int,
653 + device: str,
654 + kv_cache_dtype: str,
655 + align_cache: bool,
656 +) -> None:
657 + current_platform.seed_everything(seed)
658 + torch.set_default_device(device)
659 +
660 + entry_size = kv_lora_rank + qk_rope_head_dim
661 +
662 + src_cache = _create_mla_cache(num_blocks, block_size, entry_size, dtype,
663 + kv_cache_dtype, device, align_cache)
664 + dst_cache = _create_mla_cache(num_blocks, block_size, entry_size, dtype,
665 + kv_cache_dtype, device, align_cache)
666 +
667 + _fill_mla_cache(src_cache, kv_cache_dtype)
668 + _fill_mla_cache(dst_cache, kv_cache_dtype)
669 +
670 + src_cache_clone = src_cache.clone()
671 +
672 + num_mappings = min(2, num_blocks // 2)
673 + src_blocks = random.sample(range(num_blocks), num_mappings)
674 + remaining_blocks = list(set(range(num_blocks)) - set(src_blocks))
675 + dst_blocks = random.sample(remaining_blocks, num_mappings)
676 + block_mapping = list(zip(src_blocks, dst_blocks))
677 + block_mapping_tensor = torch.tensor(block_mapping,
678 + dtype=torch.int64,
679 + device="cpu").view(-1, 2)
680 +
681 + opcheck(
682 + torch.ops._C_cache_ops.swap_blocks,
683 + (src_cache, dst_cache, block_mapping_tensor),
684 + test_utils=DEFAULT_OPCHECK_TEST_UTILS,
685 + cond=(kv_lora_rank == KV_LORA_RANKS[0]
686 + and qk_rope_head_dim == QK_ROPE_HEAD_DIMS[0]),
687 + )
688 +
689 + ops.swap_blocks(src_cache, dst_cache, block_mapping_tensor)
690 +
691 + for src, dst in block_mapping:
692 + torch.testing.assert_close(
693 + src_cache_clone[src].cpu(),
694 + dst_cache[dst].cpu(),
695 + msg=f"Block {src} from src should have been swapped to block "
696 + f"{dst} in dst_cache.")
vllm/_custom_ops.py
+5
@@ -1037,6 +1037,11 @@ def copy_blocks(key_caches: List[torch.Tensor],
1037 torch.ops._C_cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
1038
1039
1040 +def copy_blocks_mla(kv_caches: List[torch.Tensor],
1041 + block_mapping: torch.Tensor) -> None:
1042 + torch.ops._C_cache_ops.copy_blocks_mla(kv_caches, block_mapping)
1043 +
1044 +
1045 def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
1046 block_mapping: torch.Tensor) -> None:
1047 torch.ops._C_cache_ops.swap_blocks(src, dst, block_mapping)
vllm/attention/backends/triton_mla.py
+2 -3
@@ -26,7 +26,6 @@ from vllm.attention.backends.mla.utils import MLACommonImpl, MLACommonMetadata
26 from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
27 compute_slot_mapping_start_idx,
28 is_block_tables_empty)
29 -from vllm.attention.ops.paged_attn import PagedAttention
29 from vllm.attention.ops.triton_decode_attention import decode_attention_fwd
30 from vllm.utils import async_tensor_h2d, make_tensor_with_pad
31
@@ -72,14 +71,14 @@ class TritonMLABackend(AttentionBackend):
71 dst_kv_cache: torch.Tensor,
72 src_to_dst: torch.Tensor,
73 ) -> None:
75 - PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
74 + ops.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
75
76 @staticmethod
77 def copy_blocks(
78 kv_caches: List[torch.Tensor],
79 src_to_dists: torch.Tensor,
80 ) -> None:
82 - PagedAttention.copy_blocks(kv_caches, src_to_dists)
81 + ops.copy_blocks_mla(kv_caches, src_to_dists)
82
83 @staticmethod
84 def get_supported_head_sizes() -> List[int]:
vllm/attention/ops/triton_decode_attention.py
+8 -8
@@ -204,10 +204,10 @@ def _decode_att_m_fwd(
204 Req_to_tokens.stride(0),
205 q.stride(0),
206 q.stride(1),
207 - k_buffer.stride(-2),
208 - k_buffer.stride(-1),
209 - v_buffer.stride(-2),
210 - v_buffer.stride(-1),
207 + k_buffer.stride(-3), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
208 + k_buffer.stride(-2), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
209 + v_buffer.stride(-3), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
210 + v_buffer.stride(-2), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
211 att_out.stride(0),
212 att_out.stride(1),
213 att_out.stride(2),
@@ -438,10 +438,10 @@ def _decode_grouped_att_m_fwd(
438 Req_to_tokens.stride(0),
439 q.stride(0),
440 q.stride(1),
441 - k_buffer.stride(-2),
442 - k_buffer.stride(-1),
443 - v_buffer.stride(-2),
444 - v_buffer.stride(-1),
441 + k_buffer.stride(-3), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
442 + k_buffer.stride(-2), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
443 + v_buffer.stride(-3), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
444 + v_buffer.stride(-2), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
445 att_out.stride(0),
446 att_out.stride(1),
447 att_out.stride(2),
vllm/envs.py
+10
@@ -82,6 +82,7 @@ if TYPE_CHECKING:
82 VLLM_MLA_DISABLE: bool = False
83 VLLM_MLA_PERFORM_MATRIX_ABSORPTION: bool = True
84 VLLM_MLA_DISABLE_REQUANTIZATION: bool = False
85 + VLLM_MLA_CUDA_MEM_ALIGN_KV_CACHE: bool = True
86 VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON: bool = False
87
88
@@ -539,6 +540,15 @@ environment_variables: Dict[str, Callable[[], Any]] = {
540 "VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON":
541 lambda: bool(int(os.getenv("VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON", "0"))
542 ),
543 +
544 + # When on a Nvidia GPU aligns single entries (within a page) so they are 256
545 + # byte aligned for better performance, this increases the memory usage of
546 + # the cache. Currently this only affects MLA that results in non-256
547 + # byte aligned entries. This matches the alignment the CUDA runtime uses
548 + # for all allocations. Currently this primarily affects MLA, for most other
549 + # models the alignment is already naturally aligned to 256 bytes.
550 + "VLLM_CUDA_MEM_ALIGN_KV_CACHE":
551 + lambda: bool(int(os.getenv("VLLM_CUDA_MEM_ALIGN_KV_CACHE", "1"))),
552 }
553
554 # end-env-vars-definition
vllm/utils.py
+10
@@ -563,6 +563,10 @@ def cdiv(a: int, b: int) -> int:
563 return -(a // -b)
564
565
566 +def round_up(x: int, y: int) -> int:
567 + return ((x + y - 1) // y) * y
568 +
569 +
570 def _generate_random_fp8(
571 tensor: torch.Tensor,
572 low: float,
@@ -794,6 +798,12 @@ def get_dtype_size(dtype: torch.dtype) -> int:
798 return torch.tensor([], dtype=dtype).element_size()
799
800
801 +def align_to_256bytes(extent: int, dtype: torch.dtype) -> int:
802 + dtype_size = get_dtype_size(dtype)
803 + eles_per_256bytes = 256 // dtype_size
804 + return round_up(extent, eles_per_256bytes)
805 +
806 +
807 # `collections` helpers
808 def is_list_of(
809 value: object,
vllm/worker/cache_engine.py
+55 -11
@@ -2,13 +2,17 @@
2 """CacheEngine class for managing the KV cache."""
3 from typing import List
4
5 +import numpy as np
6 import torch
7
8 +from vllm import envs
9 from vllm.attention import get_attn_backend
10 from vllm.config import CacheConfig, DeviceConfig, ModelConfig, ParallelConfig
11 from vllm.logger import init_logger
12 +from vllm.platforms import current_platform
13 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, LayerBlockType,
11 - get_dtype_size, is_pin_memory_available)
14 + align_to_256bytes, get_dtype_size,
15 + is_pin_memory_available)
16
17 logger = init_logger(__name__)
18
@@ -38,6 +42,7 @@ class CacheEngine:
42 self.num_attention_layers = model_config.get_num_layers_by_block_type(
43 parallel_config, LayerBlockType.attention)
44 self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
45 + self.align_cache = self._align_cache(model_config)
46
47 self.block_size = cache_config.block_size
48 self.num_gpu_blocks = cache_config.num_gpu_blocks
@@ -75,15 +80,39 @@ class CacheEngine:
80 num_blocks, self.block_size, self.num_kv_heads, self.head_size)
81 pin_memory = is_pin_memory_available() if device == "cpu" else False
82 kv_cache: List[torch.Tensor] = []
83 +
84 + # Align entries so they are 256 byte aligned for better performance
85 + # Primarily targets MLA as this typically only ends up having entries
86 + # be 128 byte aligned.
87 + if self.align_cache:
88 + # We assume the cache shape is:
89 + # (TOTAL_PAGES, PAGE_SIZE, entry_shape...)
90 + # NOTE this assumption currently only holds for MLA so we only apply
91 + # this optimization when `use_mla` is true
92 + entry_shape = kv_cache_shape[2:]
93 + entry_size = np.prod(entry_shape)
94 + alloc_entry_size = align_to_256bytes(entry_size, self.dtype)
95 + alloc_shape = (*kv_cache_shape[:2], alloc_entry_size)
96 + else:
97 + alloc_shape = kv_cache_shape
98 +
99 for _ in range(self.num_attention_layers):
100 # null block in CpuGpuBlockAllocator requires at least that
101 # block to be zeroed-out.
102 # We zero-out everything for simplicity.
82 - kv_cache.append(
83 - torch.zeros(kv_cache_shape,
84 - dtype=self.dtype,
85 - pin_memory=pin_memory,
86 - device=device))
103 + layer_kv_cache = torch.zeros(alloc_shape,
104 + dtype=self.dtype,
105 + pin_memory=pin_memory,
106 + device=device)
107 +
108 + # If we allocated with padding for alignment reasons truncate the
109 + # shape while preserving the aligned stride
110 + if self.align_cache:
111 + layer_kv_cache = layer_kv_cache[..., :entry_size]
112 +
113 + # view back to (TOTAL_PAGES, PAGE_SIZE, entry_shape...) for cases
114 + # when entry_shape is higher than 1D
115 + kv_cache.append(layer_kv_cache.view(kv_cache_shape))
116 return kv_cache
117
118 def swap_in(self, src_to_dst: torch.Tensor) -> None:
@@ -99,6 +128,14 @@ class CacheEngine:
128 def copy(self, src_to_dsts: torch.Tensor) -> None:
129 self.attn_backend.copy_blocks(self.gpu_cache, src_to_dsts)
130
131 + @staticmethod
132 + def _align_cache(model_config: ModelConfig):
133 + # Currently align_cache only applies to MLA models since the other
134 + # cache kernels haven't been updated yet to support non-continguous
135 + # tensors
136 + return model_config.use_mla and current_platform.is_cuda() \
137 + and envs.VLLM_CUDA_MEM_ALIGN_KV_CACHE
138 +
139 @staticmethod
140 def get_cache_block_size(
141 cache_config: CacheConfig,
@@ -110,14 +147,21 @@ class CacheEngine:
147 num_attention_layers = model_config.get_num_layers_by_block_type(
148 parallel_config, LayerBlockType.attention)
149
113 - key_cache_block = cache_config.block_size * num_heads * head_size
114 - # For MLA there is no value cache, since the latent vector
115 - # is joint keys and values.
116 - value_cache_block = key_cache_block if not model_config.use_mla else 0
117 - total = num_attention_layers * (key_cache_block + value_cache_block)
150 if cache_config.cache_dtype == "auto":
151 dtype = model_config.dtype
152 else:
153 dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
154 +
155 + key_cache_entry = num_heads * head_size
156 + if CacheEngine._align_cache(model_config):
157 + key_cache_entry = align_to_256bytes(key_cache_entry,
158 + model_config.dtype)
159 +
160 + # For MLA there is no value cache, since the latent vector
161 + # is joint keys and values.
162 + value_cache_entry = key_cache_entry if not model_config.use_mla else 0
163 + total = num_attention_layers * cache_config.block_size * \
164 + (key_cache_entry + value_cache_entry)
165 +
166 dtype_size = get_dtype_size(dtype)
167 return dtype_size * total