A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Perf] Mem align KV caches for CUDA devices (MLA perf improvement) (#12676)
Signed-off-by: simon-mo <xmo@berkeley.edu> Signed-off-by: Lucas Wilkinson <lcwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: simon-mo <xmo@berkeley.edu>
Lucas Wilkinson committed
Feb 4, 2025 at 21:22 UTC
75e94309e8d8919e0daea041f6cd81a4b8c09060
10 files changed
+429
-34
csrc/cache.h
+3
@@ -15,6 +15,9 @@ void copy_blocks(std::vector<torch::Tensor> const& key_caches,
15
std::vector<torch::Tensor> const& value_caches,
16
const torch::Tensor& block_mapping);
17
18
+void copy_blocks_mla(std::vector<torch::Tensor> const& kv_caches,
19
+ const torch::Tensor& block_mapping);
20
+
21
void reshape_and_cache(torch::Tensor& key, torch::Tensor& value,
22
torch::Tensor& key_cache, torch::Tensor& value_cache,
23
torch::Tensor& slot_mapping,
csrc/cache_kernels.cu
+70
-12
@@ -46,7 +46,10 @@ void swap_blocks(torch::Tensor& src, torch::Tensor& dst,
46
char* src_ptr = static_cast<char*>(src.data_ptr());
47
char* dst_ptr = static_cast<char*>(dst.data_ptr());
48
49
- const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
49
+ // We use the stride instead of numel in case the cache is padded for memory
50
+ // alignment reasons, we assume the blocks data (inclusive of any padding)
51
+ // is contiguous in memory
52
+ const int64_t block_size_in_bytes = src.element_size() * src.stride(0);
53
const at::cuda::OptionalCUDAGuard device_guard(
54
src_device.is_cuda() ? src_device : dst_device);
55
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
@@ -93,6 +96,24 @@ __global__ void copy_blocks_kernel(int64_t* key_cache_ptrs,
96
}
97
}
98
99
+// Kernel for MLA, which works on a single joint kv_cache
100
+// Grid: (num_layers, num_pairs)
101
+template <typename scalar_t>
102
+__global__ void copy_blocks_mla_kernel(
103
+ int64_t* cache_ptrs, const int64_t* __restrict__ block_mapping,
104
+ const int mem_footprint_per_block) {
105
+ const int layer_idx = blockIdx.x;
106
+ const int pair_idx = blockIdx.y;
107
+ scalar_t* cache = reinterpret_cast<scalar_t*>(cache_ptrs[layer_idx]);
108
+ int64_t src_block = block_mapping[2 * pair_idx];
109
+ int64_t dst_block = block_mapping[2 * pair_idx + 1];
110
+ int64_t src_offset = src_block * mem_footprint_per_block;
111
+ int64_t dst_offset = dst_block * mem_footprint_per_block;
112
+ for (int i = threadIdx.x; i < mem_footprint_per_block; i += blockDim.x) {
113
+ cache[dst_offset + i] = cache[src_offset + i];
114
+ }
115
+}
116
+
117
} // namespace vllm
118
119
// Note: the key_caches and value_caches vectors are constant but
@@ -147,6 +168,42 @@ void copy_blocks(std::vector<torch::Tensor> const& key_caches,
168
}));
169
}
170
171
+// copy blocks kernel for MLA (assumes a joint KV-cache)
172
+void copy_blocks_mla(std::vector<torch::Tensor> const& kv_caches,
173
+ const torch::Tensor& block_mapping) {
174
+ int num_layers = kv_caches.size();
175
+ if (num_layers == 0) {
176
+ return;
177
+ }
178
+ torch::Device cache_device = kv_caches[0].device();
179
+ TORCH_CHECK(cache_device.is_cuda(), "kv_cache must be on CUDA");
180
+
181
+ std::vector<int64_t> cache_ptrs(num_layers);
182
+ for (int layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
183
+ cache_ptrs[layer_idx] =
184
+ reinterpret_cast<int64_t>(kv_caches[layer_idx].data_ptr());
185
+ }
186
+ torch::Tensor cache_ptrs_tensor =
187
+ torch::from_blob(cache_ptrs.data(), {num_layers}, torch::kInt64)
188
+ .to(cache_device);
189
+
190
+ int num_pairs = block_mapping.size(0);
191
+ // We use the stride instead of numel in case the cache is padded for memory
192
+ // alignment reasons, we assume the blocks data (inclusive of any padding)
193
+ // is contiguous in memory
194
+ int mem_footprint_per_block = kv_caches[0].stride(0);
195
+ dim3 grid(num_layers, num_pairs);
196
+ dim3 block(std::min(1024, mem_footprint_per_block));
197
+ const at::cuda::OptionalCUDAGuard device_guard(cache_device);
198
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
199
+ VLLM_DISPATCH_FLOATING_AND_BYTE_TYPES(
200
+ kv_caches[0].scalar_type(), "copy_blocks_mla_kernel", ([&] {
201
+ vllm::copy_blocks_mla_kernel<scalar_t><<<grid, block, 0, stream>>>(
202
+ cache_ptrs_tensor.data_ptr<int64_t>(),
203
+ block_mapping.data_ptr<int64_t>(), mem_footprint_per_block);
204
+ }));
205
+}
206
+
207
namespace vllm {
208
209
template <typename scalar_t, typename cache_t, Fp8KVCacheDataType kv_dt>
@@ -254,6 +311,7 @@ __global__ void concat_and_cache_mla_kernel(
311
// + pe_dim)]
312
const int64_t* __restrict__ slot_mapping, // [num_tokens]
313
const int block_stride, //
314
+ const int entry_stride, //
315
const int kv_c_stride, //
316
const int k_pe_stride, //
317
const int kv_lora_rank, //
@@ -274,9 +332,8 @@ __global__ void concat_and_cache_mla_kernel(
332
int src_stride, int dst_stride, int size, int offset) {
333
for (int i = threadIdx.x; i < size; i += blockDim.x) {
334
const int64_t src_idx = token_idx * src_stride + i;
277
- const int64_t dst_idx = block_idx * block_stride +
278
- block_offset * (kv_lora_rank + pe_dim) + i +
279
- offset;
335
+ const int64_t dst_idx =
336
+ block_idx * block_stride + block_offset * entry_stride + i + offset;
337
if constexpr (kv_dt == Fp8KVCacheDataType::kAuto) {
338
dst[dst_idx] = src[src_idx];
339
} else {
@@ -391,14 +448,14 @@ void reshape_and_cache_flash(
448
// KV_T is the stored data type of kv-cache.
449
// CACHE_T is the data type of key and value tensors.
450
// KV_DTYPE is the real data type of kv-cache.
394
-#define CALL_CONCAT_AND_CACHE_MLA(KV_T, CACHE_T, KV_DTYPE) \
395
- vllm::concat_and_cache_mla_kernel<KV_T, CACHE_T, KV_DTYPE> \
396
- <<<grid, block, 0, stream>>>( \
397
- reinterpret_cast<KV_T*>(kv_c.data_ptr()), \
398
- reinterpret_cast<KV_T*>(k_pe.data_ptr()), \
399
- reinterpret_cast<CACHE_T*>(kv_cache.data_ptr()), \
400
- slot_mapping.data_ptr<int64_t>(), block_stride, kv_c_stride, \
401
- k_pe_stride, kv_lora_rank, pe_dim, block_size, \
451
+#define CALL_CONCAT_AND_CACHE_MLA(KV_T, CACHE_T, KV_DTYPE) \
452
+ vllm::concat_and_cache_mla_kernel<KV_T, CACHE_T, KV_DTYPE> \
453
+ <<<grid, block, 0, stream>>>( \
454
+ reinterpret_cast<KV_T*>(kv_c.data_ptr()), \
455
+ reinterpret_cast<KV_T*>(k_pe.data_ptr()), \
456
+ reinterpret_cast<CACHE_T*>(kv_cache.data_ptr()), \
457
+ slot_mapping.data_ptr<int64_t>(), block_stride, entry_stride, \
458
+ kv_c_stride, k_pe_stride, kv_lora_rank, pe_dim, block_size, \
459
reinterpret_cast<const float*>(scale.data_ptr()));
460
461
void concat_and_cache_mla(
@@ -428,6 +485,7 @@ void concat_and_cache_mla(
485
int kv_c_stride = kv_c.stride(0);
486
int k_pe_stride = k_pe.stride(0);
487
int block_stride = kv_cache.stride(0);
488
+ int entry_stride = kv_cache.stride(1);
489
490
dim3 grid(num_tokens);
491
dim3 block(std::min(kv_lora_rank, 512));
csrc/torch_bindings.cpp
+4
@@ -450,6 +450,10 @@ TORCH_LIBRARY_EXPAND(CONCAT(TORCH_EXTENSION_NAME, _cache_ops), cache_ops) {
450
"Tensor block_mapping) -> ()");
451
cache_ops.impl("copy_blocks", torch::kCUDA, ©_blocks);
452
453
+ cache_ops.def(
454
+ "copy_blocks_mla(Tensor(a!)[] kv_caches, Tensor block_mapping) -> ()");
455
+ cache_ops.impl("copy_blocks_mla", torch::kCUDA, ©_blocks_mla);
456
+
457
// Reshape the key and value tensors and cache them.
458
cache_ops.def(
459
"reshape_and_cache(Tensor key, Tensor value,"
tests/kernels/test_cache.py
+262
@@ -9,6 +9,7 @@ import torch
9
from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
10
from vllm import _custom_ops as ops
11
from vllm.platforms import current_platform
12
+from vllm.utils import align_to_256bytes
13
14
COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
15
DTYPES = [torch.half, torch.bfloat16, torch.float]
@@ -18,6 +19,13 @@ NUM_HEADS = [8] # Arbitrary values for testing
19
HEAD_SIZES = [64, 80, 120, 256]
20
BLOCK_SIZES = [8, 16, 32]
21
22
+# Parameters for MLA tests.
23
+KV_LORA_RANKS = [512]
24
+QK_ROPE_HEAD_DIMS = [64]
25
+NUM_TOKENS_MLA = [42]
26
+BLOCK_SIZES_MLA = [16]
27
+NUM_BLOCKS_MLA = [8]
28
+
29
# Arbitrary values for testing
30
# don't make it too large. e.g. [1024, 36000] will OOM
31
NUM_BLOCKS = [1024, 10000]
@@ -432,3 +440,257 @@ def test_fp8_e4m3_conversion(
440
ops.convert_fp8(converted_cache, cache_fp8)
441
442
torch.testing.assert_close(cache, converted_cache, atol=0.001, rtol=0.1)
443
+
444
+
445
+def _create_mla_cache(
446
+ num_blocks: int,
447
+ block_size: int,
448
+ entry_size: int,
449
+ dtype: torch.dtype,
450
+ kv_cache_dtype: str,
451
+ device: str,
452
+ align_cache: bool,
453
+) -> torch.Tensor:
454
+ cache_dtype = torch.uint8 if kv_cache_dtype == "fp8" else dtype
455
+
456
+ if align_cache:
457
+ alloc_entry_size = align_to_256bytes(entry_size, cache_dtype)
458
+ alloc_shape = (num_blocks, block_size, alloc_entry_size)
459
+ cache_full = torch.zeros(alloc_shape, dtype=cache_dtype, device=device)
460
+ cache = cache_full[..., :entry_size]
461
+ else:
462
+ cache = torch.zeros(num_blocks,
463
+ block_size,
464
+ entry_size,
465
+ dtype=cache_dtype,
466
+ device=device)
467
+ return cache
468
+
469
+
470
+def _fill_mla_cache(cache: torch.Tensor, kv_cache_dtype: str):
471
+ rand_dtype = torch.float16 if kv_cache_dtype == "fp8" else cache.dtype
472
+
473
+ vals = torch.randn(*cache.shape, device=cache.device, dtype=rand_dtype)
474
+ if kv_cache_dtype == "fp8":
475
+ temp = torch.zeros_like(cache)
476
+ ops.convert_fp8(temp, vals, 1.0, kv_dtype=kv_cache_dtype)
477
+ vals = temp
478
+ cache.copy_(vals)
479
+
480
+
481
+@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
482
+@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
483
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS_MLA)
484
+@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
485
+@pytest.mark.parametrize("num_blocks", NUM_BLOCKS_MLA)
486
+@pytest.mark.parametrize("dtype", DTYPES)
487
+@pytest.mark.parametrize("seed", SEEDS)
488
+@pytest.mark.parametrize("device", CUDA_DEVICES)
489
+@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
490
+@pytest.mark.parametrize("align_cache", [False])
491
+@torch.inference_mode()
492
+def test_concat_and_cache_mla(
493
+ kv_lora_rank: int,
494
+ qk_rope_head_dim: int,
495
+ num_tokens: int,
496
+ block_size: int,
497
+ num_blocks: int,
498
+ dtype: torch.dtype,
499
+ seed: int,
500
+ device: str,
501
+ kv_cache_dtype: str,
502
+ align_cache: bool,
503
+) -> None:
504
+ current_platform.seed_everything(seed)
505
+ torch.set_default_device(device)
506
+
507
+ total_slots = num_blocks * block_size
508
+ slot_mapping_lst = random.sample(range(total_slots), num_tokens)
509
+ slot_mapping = torch.tensor(slot_mapping_lst,
510
+ dtype=torch.long,
511
+ device=device)
512
+
513
+ kv_c = torch.randn(num_tokens, kv_lora_rank, dtype=dtype, device=device)
514
+ k_pe = torch.randn(num_tokens,
515
+ qk_rope_head_dim,
516
+ dtype=dtype,
517
+ device=device)
518
+ entry_size = kv_lora_rank + qk_rope_head_dim
519
+
520
+ scale = torch.tensor(0.1, dtype=torch.float32, device=device)
521
+ kv_cache = _create_mla_cache(num_blocks, block_size, entry_size, dtype,
522
+ kv_cache_dtype, device, align_cache)
523
+ ref_temp = torch.zeros(*kv_cache.shape, dtype=dtype, device=device)
524
+
525
+ for i in range(num_tokens):
526
+ slot = slot_mapping[i].item()
527
+ block_idx = slot // block_size
528
+ block_offset = slot % block_size
529
+ ref_temp[block_idx, block_offset, :kv_lora_rank] = kv_c[i]
530
+ ref_temp[block_idx, block_offset, kv_lora_rank:] = k_pe[i]
531
+
532
+ if kv_cache_dtype == "fp8":
533
+ ref_kv_cache = torch.empty_like(ref_temp, dtype=kv_cache.dtype)
534
+ ops.convert_fp8(ref_kv_cache,
535
+ ref_temp,
536
+ scale.item(),
537
+ kv_dtype=kv_cache_dtype)
538
+ else:
539
+ ref_kv_cache = ref_temp
540
+
541
+ opcheck(
542
+ torch.ops._C_cache_ops.concat_and_cache_mla,
543
+ (kv_c, k_pe, kv_cache, slot_mapping, kv_cache_dtype, scale),
544
+ test_utils=DEFAULT_OPCHECK_TEST_UTILS,
545
+ )
546
+
547
+ ops.concat_and_cache_mla(kv_c, k_pe, kv_cache, slot_mapping,
548
+ kv_cache_dtype, scale)
549
+
550
+ if kv_cache_dtype == "fp8":
551
+ result_temp = torch.empty_like(kv_cache, dtype=torch.float16)
552
+ ops.convert_fp8(result_temp,
553
+ kv_cache.contiguous(),
554
+ scale.item(),
555
+ kv_dtype=kv_cache_dtype)
556
+ expected_temp = torch.empty_like(ref_kv_cache, dtype=torch.float16)
557
+ ops.convert_fp8(expected_temp,
558
+ ref_kv_cache,
559
+ scale.item(),
560
+ kv_dtype=kv_cache_dtype)
561
+ torch.testing.assert_close(result_temp,
562
+ expected_temp,
563
+ atol=0.001,
564
+ rtol=0.1)
565
+ else:
566
+ torch.testing.assert_close(kv_cache, ref_kv_cache)
567
+
568
+
569
+@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
570
+@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
571
+@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
572
+@pytest.mark.parametrize("num_blocks", NUM_BLOCKS_MLA)
573
+@pytest.mark.parametrize("num_layers", NUM_LAYERS)
574
+@pytest.mark.parametrize("dtype", DTYPES)
575
+@pytest.mark.parametrize("seed", SEEDS)
576
+@pytest.mark.parametrize("device", CUDA_DEVICES)
577
+@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
578
+@pytest.mark.parametrize("align_cache", [False, True])
579
+@torch.inference_mode()
580
+def test_copy_blocks_mla(
581
+ kv_lora_rank: int,
582
+ qk_rope_head_dim: int,
583
+ block_size: int,
584
+ num_blocks: int,
585
+ num_layers: int,
586
+ dtype: torch.dtype,
587
+ seed: int,
588
+ device: str,
589
+ kv_cache_dtype: str,
590
+ align_cache: bool,
591
+) -> None:
592
+ current_platform.seed_everything(seed)
593
+ torch.set_default_device(device)
594
+
595
+ entry_size = kv_lora_rank + qk_rope_head_dim
596
+
597
+ kv_caches = []
598
+ for _ in range(num_layers):
599
+ kv_cache = _create_mla_cache(num_blocks, block_size, entry_size, dtype,
600
+ kv_cache_dtype, device, align_cache)
601
+ _fill_mla_cache(kv_cache, kv_cache_dtype=kv_cache_dtype)
602
+ kv_caches.append(kv_cache)
603
+
604
+ ref_caches = [kv_cache.clone() for kv_cache in kv_caches]
605
+
606
+ num_mappings = min(2, num_blocks // 2)
607
+ src_blocks = random.sample(range(num_blocks), num_mappings)
608
+ remaining = list(set(range(num_blocks)) - set(src_blocks))
609
+ dst_blocks = random.sample(remaining, 2 * num_mappings)
610
+ block_mapping = []
611
+ for i in range(num_mappings):
612
+ src = src_blocks[i]
613
+ dst1 = dst_blocks[2 * i]
614
+ dst2 = dst_blocks[2 * i + 1]
615
+ block_mapping.append((src, dst1))
616
+ block_mapping.append((src, dst2))
617
+ block_mapping_tensor = torch.tensor(block_mapping,
618
+ dtype=torch.int64,
619
+ device=device).view(-1, 2)
620
+
621
+ for src, dst in block_mapping:
622
+ for ref_cache in ref_caches:
623
+ ref_cache[dst].copy_(ref_cache[src])
624
+
625
+ opcheck(
626
+ torch.ops._C_cache_ops.copy_blocks_mla,
627
+ (kv_caches, block_mapping_tensor),
628
+ test_utils=DEFAULT_OPCHECK_TEST_UTILS,
629
+ )
630
+ ops.copy_blocks_mla(kv_caches, block_mapping_tensor)
631
+
632
+ for kv_cache, ref_cache in zip(kv_caches, ref_caches):
633
+ torch.testing.assert_close(kv_cache, ref_cache)
634
+
635
+
636
+@pytest.mark.parametrize("kv_lora_rank", KV_LORA_RANKS)
637
+@pytest.mark.parametrize("qk_rope_head_dim", QK_ROPE_HEAD_DIMS)
638
+@pytest.mark.parametrize("block_size", BLOCK_SIZES_MLA)
639
+@pytest.mark.parametrize("num_blocks", NUM_BLOCKS_MLA)
640
+@pytest.mark.parametrize("dtype", DTYPES)
641
+@pytest.mark.parametrize("seed", SEEDS)
642
+@pytest.mark.parametrize("device", CUDA_DEVICES)
643
+@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
644
+@pytest.mark.parametrize("align_cache", [False, True])
645
+@torch.inference_mode()
646
+def test_swap_blocks_mla(
647
+ kv_lora_rank: int,
648
+ qk_rope_head_dim: int,
649
+ block_size: int,
650
+ num_blocks: int,
651
+ dtype: torch.dtype,
652
+ seed: int,
653
+ device: str,
654
+ kv_cache_dtype: str,
655
+ align_cache: bool,
656
+) -> None:
657
+ current_platform.seed_everything(seed)
658
+ torch.set_default_device(device)
659
+
660
+ entry_size = kv_lora_rank + qk_rope_head_dim
661
+
662
+ src_cache = _create_mla_cache(num_blocks, block_size, entry_size, dtype,
663
+ kv_cache_dtype, device, align_cache)
664
+ dst_cache = _create_mla_cache(num_blocks, block_size, entry_size, dtype,
665
+ kv_cache_dtype, device, align_cache)
666
+
667
+ _fill_mla_cache(src_cache, kv_cache_dtype)
668
+ _fill_mla_cache(dst_cache, kv_cache_dtype)
669
+
670
+ src_cache_clone = src_cache.clone()
671
+
672
+ num_mappings = min(2, num_blocks // 2)
673
+ src_blocks = random.sample(range(num_blocks), num_mappings)
674
+ remaining_blocks = list(set(range(num_blocks)) - set(src_blocks))
675
+ dst_blocks = random.sample(remaining_blocks, num_mappings)
676
+ block_mapping = list(zip(src_blocks, dst_blocks))
677
+ block_mapping_tensor = torch.tensor(block_mapping,
678
+ dtype=torch.int64,
679
+ device="cpu").view(-1, 2)
680
+
681
+ opcheck(
682
+ torch.ops._C_cache_ops.swap_blocks,
683
+ (src_cache, dst_cache, block_mapping_tensor),
684
+ test_utils=DEFAULT_OPCHECK_TEST_UTILS,
685
+ cond=(kv_lora_rank == KV_LORA_RANKS[0]
686
+ and qk_rope_head_dim == QK_ROPE_HEAD_DIMS[0]),
687
+ )
688
+
689
+ ops.swap_blocks(src_cache, dst_cache, block_mapping_tensor)
690
+
691
+ for src, dst in block_mapping:
692
+ torch.testing.assert_close(
693
+ src_cache_clone[src].cpu(),
694
+ dst_cache[dst].cpu(),
695
+ msg=f"Block {src} from src should have been swapped to block "
696
+ f"{dst} in dst_cache.")
vllm/_custom_ops.py
+5
@@ -1037,6 +1037,11 @@ def copy_blocks(key_caches: List[torch.Tensor],
1037
torch.ops._C_cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
1038
1039
1040
+def copy_blocks_mla(kv_caches: List[torch.Tensor],
1041
+ block_mapping: torch.Tensor) -> None:
1042
+ torch.ops._C_cache_ops.copy_blocks_mla(kv_caches, block_mapping)
1043
+
1044
+
1045
def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
1046
block_mapping: torch.Tensor) -> None:
1047
torch.ops._C_cache_ops.swap_blocks(src, dst, block_mapping)
vllm/attention/backends/triton_mla.py
+2
-3
@@ -26,7 +26,6 @@ from vllm.attention.backends.mla.utils import MLACommonImpl, MLACommonMetadata
26
from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
27
compute_slot_mapping_start_idx,
28
is_block_tables_empty)
29
-from vllm.attention.ops.paged_attn import PagedAttention
29
from vllm.attention.ops.triton_decode_attention import decode_attention_fwd
30
from vllm.utils import async_tensor_h2d, make_tensor_with_pad
31
@@ -72,14 +71,14 @@ class TritonMLABackend(AttentionBackend):
71
dst_kv_cache: torch.Tensor,
72
src_to_dst: torch.Tensor,
73
) -> None:
75
- PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
74
+ ops.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
75
76
@staticmethod
77
def copy_blocks(
78
kv_caches: List[torch.Tensor],
79
src_to_dists: torch.Tensor,
80
) -> None:
82
- PagedAttention.copy_blocks(kv_caches, src_to_dists)
81
+ ops.copy_blocks_mla(kv_caches, src_to_dists)
82
83
@staticmethod
84
def get_supported_head_sizes() -> List[int]:
vllm/attention/ops/triton_decode_attention.py
+8
-8
@@ -204,10 +204,10 @@ def _decode_att_m_fwd(
204
Req_to_tokens.stride(0),
205
q.stride(0),
206
q.stride(1),
207
- k_buffer.stride(-2),
208
- k_buffer.stride(-1),
209
- v_buffer.stride(-2),
210
- v_buffer.stride(-1),
207
+ k_buffer.stride(-3), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
208
+ k_buffer.stride(-2), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
209
+ v_buffer.stride(-3), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
210
+ v_buffer.stride(-2), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
211
att_out.stride(0),
212
att_out.stride(1),
213
att_out.stride(2),
@@ -438,10 +438,10 @@ def _decode_grouped_att_m_fwd(
438
Req_to_tokens.stride(0),
439
q.stride(0),
440
q.stride(1),
441
- k_buffer.stride(-2),
442
- k_buffer.stride(-1),
443
- v_buffer.stride(-2),
444
- v_buffer.stride(-1),
441
+ k_buffer.stride(-3), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
442
+ k_buffer.stride(-2), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
443
+ v_buffer.stride(-3), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
444
+ v_buffer.stride(-2), # Assume (..., PAGE_SIZE, NUM_HEADS, HEAD_DIM)
445
att_out.stride(0),
446
att_out.stride(1),
447
att_out.stride(2),
vllm/envs.py
+10
@@ -82,6 +82,7 @@ if TYPE_CHECKING:
82
VLLM_MLA_DISABLE: bool = False
83
VLLM_MLA_PERFORM_MATRIX_ABSORPTION: bool = True
84
VLLM_MLA_DISABLE_REQUANTIZATION: bool = False
85
+ VLLM_MLA_CUDA_MEM_ALIGN_KV_CACHE: bool = True
86
VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON: bool = False
87
88
@@ -539,6 +540,15 @@ environment_variables: Dict[str, Callable[[], Any]] = {
540
"VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON":
541
lambda: bool(int(os.getenv("VLLM_ENABLE_MOE_ALIGN_BLOCK_SIZE_TRITON", "0"))
542
),
543
+
544
+ # When on a Nvidia GPU aligns single entries (within a page) so they are 256
545
+ # byte aligned for better performance, this increases the memory usage of
546
+ # the cache. Currently this only affects MLA that results in non-256
547
+ # byte aligned entries. This matches the alignment the CUDA runtime uses
548
+ # for all allocations. Currently this primarily affects MLA, for most other
549
+ # models the alignment is already naturally aligned to 256 bytes.
550
+ "VLLM_CUDA_MEM_ALIGN_KV_CACHE":
551
+ lambda: bool(int(os.getenv("VLLM_CUDA_MEM_ALIGN_KV_CACHE", "1"))),
552
}
553
554
# end-env-vars-definition
vllm/utils.py
+10
@@ -563,6 +563,10 @@ def cdiv(a: int, b: int) -> int:
563
return -(a // -b)
564
565
566
+def round_up(x: int, y: int) -> int:
567
+ return ((x + y - 1) // y) * y
568
+
569
+
570
def _generate_random_fp8(
571
tensor: torch.Tensor,
572
low: float,
@@ -794,6 +798,12 @@ def get_dtype_size(dtype: torch.dtype) -> int:
798
return torch.tensor([], dtype=dtype).element_size()
799
800
801
+def align_to_256bytes(extent: int, dtype: torch.dtype) -> int:
802
+ dtype_size = get_dtype_size(dtype)
803
+ eles_per_256bytes = 256 // dtype_size
804
+ return round_up(extent, eles_per_256bytes)
805
+
806
+
807
# `collections` helpers
808
def is_list_of(
809
value: object,
vllm/worker/cache_engine.py
+55
-11
@@ -2,13 +2,17 @@
2
"""CacheEngine class for managing the KV cache."""
3
from typing import List
4
5
+import numpy as np
6
import torch
7
8
+from vllm import envs
9
from vllm.attention import get_attn_backend
10
from vllm.config import CacheConfig, DeviceConfig, ModelConfig, ParallelConfig
11
from vllm.logger import init_logger
12
+from vllm.platforms import current_platform
13
from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, LayerBlockType,
11
- get_dtype_size, is_pin_memory_available)
14
+ align_to_256bytes, get_dtype_size,
15
+ is_pin_memory_available)
16
17
logger = init_logger(__name__)
18
@@ -38,6 +42,7 @@ class CacheEngine:
42
self.num_attention_layers = model_config.get_num_layers_by_block_type(
43
parallel_config, LayerBlockType.attention)
44
self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
45
+ self.align_cache = self._align_cache(model_config)
46
47
self.block_size = cache_config.block_size
48
self.num_gpu_blocks = cache_config.num_gpu_blocks
@@ -75,15 +80,39 @@ class CacheEngine:
80
num_blocks, self.block_size, self.num_kv_heads, self.head_size)
81
pin_memory = is_pin_memory_available() if device == "cpu" else False
82
kv_cache: List[torch.Tensor] = []
83
+
84
+ # Align entries so they are 256 byte aligned for better performance
85
+ # Primarily targets MLA as this typically only ends up having entries
86
+ # be 128 byte aligned.
87
+ if self.align_cache:
88
+ # We assume the cache shape is:
89
+ # (TOTAL_PAGES, PAGE_SIZE, entry_shape...)
90
+ # NOTE this assumption currently only holds for MLA so we only apply
91
+ # this optimization when `use_mla` is true
92
+ entry_shape = kv_cache_shape[2:]
93
+ entry_size = np.prod(entry_shape)
94
+ alloc_entry_size = align_to_256bytes(entry_size, self.dtype)
95
+ alloc_shape = (*kv_cache_shape[:2], alloc_entry_size)
96
+ else:
97
+ alloc_shape = kv_cache_shape
98
+
99
for _ in range(self.num_attention_layers):
100
# null block in CpuGpuBlockAllocator requires at least that
101
# block to be zeroed-out.
102
# We zero-out everything for simplicity.
82
- kv_cache.append(
83
- torch.zeros(kv_cache_shape,
84
- dtype=self.dtype,
85
- pin_memory=pin_memory,
86
- device=device))
103
+ layer_kv_cache = torch.zeros(alloc_shape,
104
+ dtype=self.dtype,
105
+ pin_memory=pin_memory,
106
+ device=device)
107
+
108
+ # If we allocated with padding for alignment reasons truncate the
109
+ # shape while preserving the aligned stride
110
+ if self.align_cache:
111
+ layer_kv_cache = layer_kv_cache[..., :entry_size]
112
+
113
+ # view back to (TOTAL_PAGES, PAGE_SIZE, entry_shape...) for cases
114
+ # when entry_shape is higher than 1D
115
+ kv_cache.append(layer_kv_cache.view(kv_cache_shape))
116
return kv_cache
117
118
def swap_in(self, src_to_dst: torch.Tensor) -> None:
@@ -99,6 +128,14 @@ class CacheEngine:
128
def copy(self, src_to_dsts: torch.Tensor) -> None:
129
self.attn_backend.copy_blocks(self.gpu_cache, src_to_dsts)
130
131
+ @staticmethod
132
+ def _align_cache(model_config: ModelConfig):
133
+ # Currently align_cache only applies to MLA models since the other
134
+ # cache kernels haven't been updated yet to support non-continguous
135
+ # tensors
136
+ return model_config.use_mla and current_platform.is_cuda() \
137
+ and envs.VLLM_CUDA_MEM_ALIGN_KV_CACHE
138
+
139
@staticmethod
140
def get_cache_block_size(
141
cache_config: CacheConfig,
@@ -110,14 +147,21 @@ class CacheEngine:
147
num_attention_layers = model_config.get_num_layers_by_block_type(
148
parallel_config, LayerBlockType.attention)
149
113
- key_cache_block = cache_config.block_size * num_heads * head_size
114
- # For MLA there is no value cache, since the latent vector
115
- # is joint keys and values.
116
- value_cache_block = key_cache_block if not model_config.use_mla else 0
117
- total = num_attention_layers * (key_cache_block + value_cache_block)
150
if cache_config.cache_dtype == "auto":
151
dtype = model_config.dtype
152
else:
153
dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
154
+
155
+ key_cache_entry = num_heads * head_size
156
+ if CacheEngine._align_cache(model_config):
157
+ key_cache_entry = align_to_256bytes(key_cache_entry,
158
+ model_config.dtype)
159
+
160
+ # For MLA there is no value cache, since the latent vector
161
+ # is joint keys and values.
162
+ value_cache_entry = key_cache_entry if not model_config.use_mla else 0
163
+ total = num_attention_layers * cache_config.block_size * \
164
+ (key_cache_entry + value_cache_entry)
165
+
166
dtype_size = get_dtype_size(dtype)
167
return dtype_size * total