A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Implement cache ops
Woosuk Kwon committed
Feb 16, 2023 at 07:47 UTC
6f058c7ba88e657457ad5db9226d5b194e5aaabe
4 files changed
+109
-6
cacheflow/worker/cache_engine.py
+23
-6
@@ -1,6 +1,7 @@
1
from typing import Dict, List, Tuple
2
3
import torch
4
+from cacheflow import ops
5
6
KVCache = Tuple[torch.Tensor, torch.Tensor]
7
@@ -92,14 +93,30 @@ class CacheEngine:
93
cpu_cache.append((key_blocks, value_blocks))
94
return cpu_cache
95
96
+ def _copy_blocks(
97
+ self,
98
+ src: List[KVCache],
99
+ dst: List[KVCache],
100
+ src_to_dst: Dict[int, int],
101
+ ) -> None:
102
+ with torch.cuda.stream(self.cache_stream):
103
+ for i in range(self.num_layers):
104
+ src_key_cache, src_value_cache = src[i]
105
+ dst_key_cache, dst_value_cache = dst[i]
106
+ # Copy the key blocks.
107
+ ops.copy_cache_blocks(
108
+ src_key_cache, dst_key_cache, src_to_dst)
109
+ # Copy the value blocks.
110
+ ops.copy_cache_blocks(
111
+ src_value_cache, dst_value_cache, src_to_dst)
112
+ event = self.events[i]
113
+ event.record(stream=self.cache_stream)
114
+
115
def copy(self, src_to_dst: Dict[int, int]) -> None:
96
- for event in self.events:
97
- pass
116
+ self._copy_blocks(self.gpu_cache, self.gpu_cache, src_to_dst)
117
118
def swap_in(self, src_to_dst: Dict[int, int]) -> None:
100
- for event in self.events:
101
- pass
119
+ self._copy_blocks(self.cpu_cache, self.gpu_cache, src_to_dst)
120
121
def swap_out(self, src_to_dst: Dict[int, int]) -> None:
104
- for event in self.events:
105
- pass
122
+ self._copy_blocks(self.gpu_cache, self.cpu_cache, src_to_dst)
csrc/cache.cpp
new
+20
@@ -0,0 +1,20 @@
1
+#include <torch/extension.h>
2
+
3
+void copy_blocks(
4
+ torch::Tensor& src,
5
+ torch::Tensor& dst,
6
+ const std::map<int64_t, int64_t>& block_mapping);
7
+
8
+void copy_cache_blocks(
9
+ torch::Tensor& src,
10
+ torch::Tensor& dst,
11
+ const std::map<int64_t, int64_t>& block_mapping) {
12
+ copy_blocks(src, dst, block_mapping);
13
+}
14
+
15
+PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
16
+ m.def(
17
+ "copy_cache_blocks",
18
+ ©_cache_blocks,
19
+ "Copy the cache blocks from src to dst");
20
+}
csrc/cache_kernel.cu
new
+43
@@ -0,0 +1,43 @@
1
+#include <torch/extension.h>
2
+
3
+#include <ATen/cuda/CUDAContext.h>
4
+
5
+#include <cassert>
6
+#include <map>
7
+
8
+void copy_blocks(
9
+ torch::Tensor& src,
10
+ torch::Tensor& dst,
11
+ const std::map<int64_t, int64_t>& block_mapping) {
12
+ torch::Device src_device = src.device();
13
+ torch::Device dst_device = dst.device();
14
+ cudaMemcpyKind memcpy_type;
15
+ if (src_device.is_cuda() && dst_device.is_cuda()) {
16
+ assert(src_device.index() == dst_device.index());
17
+ memcpy_type = cudaMemcpyDeviceToDevice;
18
+ } else if (src_device.is_cuda() && dst_device.is_cpu()) {
19
+ memcpy_type = cudaMemcpyDeviceToHost;
20
+ } else if (src_device.is_cpu() && dst_device.is_cuda()) {
21
+ memcpy_type = cudaMemcpyHostToDevice;
22
+ } else {
23
+ assert(false);
24
+ }
25
+
26
+ void *src_ptr = src.data_ptr();
27
+ void *dst_ptr = dst.data_ptr();
28
+
29
+ const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
30
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
31
+ for (const auto& pair : block_mapping) {
32
+ int64_t src_block_number = pair.first;
33
+ int64_t dst_block_number = pair.second;
34
+ int64_t src_offset = src_block_number * block_size_in_bytes;
35
+ int64_t dst_offset = dst_block_number * block_size_in_bytes;
36
+ cudaMemcpyAsync(
37
+ dst_ptr + dst_offset,
38
+ src_ptr + src_offset,
39
+ block_size_in_bytes,
40
+ memcpy_type,
41
+ stream);
42
+ }
43
+}
setup.py
+23
@@ -0,0 +1,23 @@
1
+import setuptools
2
+from torch.utils import cpp_extension
3
+
4
+CXX_FLAGS = ['-g']
5
+NVCC_FLAGS = ['-O2']
6
+
7
+
8
+ext_modules = []
9
+
10
+# Cache operations.
11
+cache_extension = cpp_extension.CUDAExtension(
12
+ name='cacheflow.ops',
13
+ sources=['csrc/cache.cpp', 'csrc/cache_kernel.cu'],
14
+ extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
15
+)
16
+ext_modules.append(cache_extension)
17
+
18
+setuptools.setup(
19
+ name='cacheflow',
20
+ requires_python='>=3.9',
21
+ ext_modules=ext_modules,
22
+ cmdclass={'build_ext': cpp_extension.BuildExtension},
23
+)