A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add reshape_and_cache op
Woosuk Kwon committed
Feb 18, 2023 at 19:22 UTC
c413c41cda0f9359e7a12bb674c0f87bf41798c5
2 files changed
+83
csrc/cache.cpp
+11
@@ -5,9 +5,20 @@ void copy_blocks(
5
torch::Tensor& dst,
6
const std::map<int64_t, int64_t>& block_mapping);
7
8
+void reshape_and_cache(
9
+ torch::Tensor& key,
10
+ torch::Tensor& value,
11
+ torch::Tensor& key_cache,
12
+ torch::Tensor& value_cache,
13
+ torch::Tensor& slot_mapping);
14
+
15
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
16
m.def(
17
"copy_cache_blocks",
18
©_blocks,
19
"Copy the cache blocks from src to dst");
20
+ m.def(
21
+ "reshape_and_cache",
22
+ &reshape_and_cache,
23
+ "Reshape the key and value tensors and cache them");
24
}
csrc/cache_kernels.cu
+72
@@ -2,6 +2,7 @@
2
3
#include <ATen/cuda/CUDAContext.h>
4
5
+#include <algorithm>
6
#include <cassert>
7
#include <map>
8
@@ -41,3 +42,74 @@ void copy_blocks(
42
stream);
43
}
44
}
45
+
46
+template<typename scalar_t>
47
+__global__ void reshape_and_cache_kernel(
48
+ const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
49
+ const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
50
+ scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
51
+ scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, block_size, head_size]
52
+ const int* __restrict__ slot_mapping, // [num_tokens]
53
+ const int num_heads,
54
+ const int head_size,
55
+ const int block_size,
56
+ const int x) {
57
+ const int token_idx = blockIdx.x;
58
+ const int slot_idx = slot_mapping[token_idx];
59
+ const int block_idx = slot_idx / block_size;
60
+ const int block_offset = slot_idx % block_size;
61
+
62
+ const int n = num_heads * head_size;
63
+ for (int i = threadIdx.x; i < n; i += blockDim.x) {
64
+ const int src_idx = token_idx * n + i;
65
+
66
+ const int head_idx = i / head_size;
67
+ const int head_offset = i % head_size;
68
+ const int x_idx = head_offset / x;
69
+ const int x_offset = head_offset % x;
70
+
71
+ const int tgt_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
72
+ + head_idx * (head_size / x) * block_size * x
73
+ + x_idx * block_size * x
74
+ + block_offset * x
75
+ + x_offset;
76
+ const int tgt_value_idx = block_idx * num_heads * block_size * head_size
77
+ + head_idx * block_size * head_size
78
+ + block_offset * head_size
79
+ + head_offset;
80
+ key_cache[tgt_key_idx] = __ldg(&key[src_idx]);
81
+ value_cache[tgt_value_idx] = __ldg(&value[src_idx]);
82
+ }
83
+}
84
+
85
+void reshape_and_cache(
86
+ torch::Tensor& key,
87
+ torch::Tensor& value,
88
+ torch::Tensor& key_cache,
89
+ torch::Tensor& value_cache,
90
+ torch::Tensor& slot_mapping) {
91
+ int num_tokens = key.size(0);
92
+ int head_num = key.size(1);
93
+ int head_size = key.size(2);
94
+ int block_size = key_cache.size(3);
95
+ int x = key_cache.size(4);
96
+
97
+ dim3 grid(num_tokens);
98
+ dim3 block(std::min(head_num * head_size, 512));
99
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
100
+ AT_DISPATCH_FLOATING_TYPES_AND_HALF(
101
+ key.scalar_type(),
102
+ "reshape_and_cache_kernel",
103
+ [&] {
104
+ reshape_and_cache_kernel<scalar_t><<<grid, block, 0, stream>>>(
105
+ key.data_ptr<scalar_t>(),
106
+ value.data_ptr<scalar_t>(),
107
+ key_cache.data_ptr<scalar_t>(),
108
+ value_cache.data_ptr<scalar_t>(),
109
+ slot_mapping.data_ptr<int>(),
110
+ head_num,
111
+ head_size,
112
+ block_size,
113
+ x);
114
+ });
115
+}