@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c413c41cda
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add reshape_and_cache op

Woosuk Kwon committed Feb 18, 2023 at 19:22 UTC c413c41cda0f9359e7a12bb674c0f87bf41798c5
2 files changed +83
csrc/cache.cpp
+11
@@ -5,9 +5,20 @@ void copy_blocks(
5 torch::Tensor& dst,
6 const std::map<int64_t, int64_t>& block_mapping);
7
8 +void reshape_and_cache(
9 + torch::Tensor& key,
10 + torch::Tensor& value,
11 + torch::Tensor& key_cache,
12 + torch::Tensor& value_cache,
13 + torch::Tensor& slot_mapping);
14 +
15 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
16 m.def(
17 "copy_cache_blocks",
18 &copy_blocks,
19 "Copy the cache blocks from src to dst");
20 + m.def(
21 + "reshape_and_cache",
22 + &reshape_and_cache,
23 + "Reshape the key and value tensors and cache them");
24 }
csrc/cache_kernels.cu
+72
@@ -2,6 +2,7 @@
2
3 #include <ATen/cuda/CUDAContext.h>
4
5 +#include <algorithm>
6 #include <cassert>
7 #include <map>
8
@@ -41,3 +42,74 @@ void copy_blocks(
42 stream);
43 }
44 }
45 +
46 +template<typename scalar_t>
47 +__global__ void reshape_and_cache_kernel(
48 + const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
49 + const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
50 + scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
51 + scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, block_size, head_size]
52 + const int* __restrict__ slot_mapping, // [num_tokens]
53 + const int num_heads,
54 + const int head_size,
55 + const int block_size,
56 + const int x) {
57 + const int token_idx = blockIdx.x;
58 + const int slot_idx = slot_mapping[token_idx];
59 + const int block_idx = slot_idx / block_size;
60 + const int block_offset = slot_idx % block_size;
61 +
62 + const int n = num_heads * head_size;
63 + for (int i = threadIdx.x; i < n; i += blockDim.x) {
64 + const int src_idx = token_idx * n + i;
65 +
66 + const int head_idx = i / head_size;
67 + const int head_offset = i % head_size;
68 + const int x_idx = head_offset / x;
69 + const int x_offset = head_offset % x;
70 +
71 + const int tgt_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
72 + + head_idx * (head_size / x) * block_size * x
73 + + x_idx * block_size * x
74 + + block_offset * x
75 + + x_offset;
76 + const int tgt_value_idx = block_idx * num_heads * block_size * head_size
77 + + head_idx * block_size * head_size
78 + + block_offset * head_size
79 + + head_offset;
80 + key_cache[tgt_key_idx] = __ldg(&key[src_idx]);
81 + value_cache[tgt_value_idx] = __ldg(&value[src_idx]);
82 + }
83 +}
84 +
85 +void reshape_and_cache(
86 + torch::Tensor& key,
87 + torch::Tensor& value,
88 + torch::Tensor& key_cache,
89 + torch::Tensor& value_cache,
90 + torch::Tensor& slot_mapping) {
91 + int num_tokens = key.size(0);
92 + int head_num = key.size(1);
93 + int head_size = key.size(2);
94 + int block_size = key_cache.size(3);
95 + int x = key_cache.size(4);
96 +
97 + dim3 grid(num_tokens);
98 + dim3 block(std::min(head_num * head_size, 512));
99 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
100 + AT_DISPATCH_FLOATING_TYPES_AND_HALF(
101 + key.scalar_type(),
102 + "reshape_and_cache_kernel",
103 + [&] {
104 + reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
105 + key.data_ptr<scalar_t>(),
106 + value.data_ptr<scalar_t>(),
107 + key_cache.data_ptr<scalar_t>(),
108 + value_cache.data_ptr<scalar_t>(),
109 + slot_mapping.data_ptr<int>(),
110 + head_num,
111 + head_size,
112 + block_size,
113 + x);
114 + });
115 +}