@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6f058c7ba8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Implement cache ops

Woosuk Kwon committed Feb 16, 2023 at 07:47 UTC 6f058c7ba88e657457ad5db9226d5b194e5aaabe
4 files changed +109 -6
cacheflow/worker/cache_engine.py
+23 -6
@@ -1,6 +1,7 @@
1 from typing import Dict, List, Tuple
2
3 import torch
4 +from cacheflow import ops
5
6 KVCache = Tuple[torch.Tensor, torch.Tensor]
7
@@ -92,14 +93,30 @@ class CacheEngine:
93 cpu_cache.append((key_blocks, value_blocks))
94 return cpu_cache
95
96 + def _copy_blocks(
97 + self,
98 + src: List[KVCache],
99 + dst: List[KVCache],
100 + src_to_dst: Dict[int, int],
101 + ) -> None:
102 + with torch.cuda.stream(self.cache_stream):
103 + for i in range(self.num_layers):
104 + src_key_cache, src_value_cache = src[i]
105 + dst_key_cache, dst_value_cache = dst[i]
106 + # Copy the key blocks.
107 + ops.copy_cache_blocks(
108 + src_key_cache, dst_key_cache, src_to_dst)
109 + # Copy the value blocks.
110 + ops.copy_cache_blocks(
111 + src_value_cache, dst_value_cache, src_to_dst)
112 + event = self.events[i]
113 + event.record(stream=self.cache_stream)
114 +
115 def copy(self, src_to_dst: Dict[int, int]) -> None:
96 - for event in self.events:
97 - pass
116 + self._copy_blocks(self.gpu_cache, self.gpu_cache, src_to_dst)
117
118 def swap_in(self, src_to_dst: Dict[int, int]) -> None:
100 - for event in self.events:
101 - pass
119 + self._copy_blocks(self.cpu_cache, self.gpu_cache, src_to_dst)
120
121 def swap_out(self, src_to_dst: Dict[int, int]) -> None:
104 - for event in self.events:
105 - pass
122 + self._copy_blocks(self.gpu_cache, self.cpu_cache, src_to_dst)
csrc/cache.cpp new
+20
@@ -0,0 +1,20 @@
1 +#include <torch/extension.h>
2 +
3 +void copy_blocks(
4 + torch::Tensor& src,
5 + torch::Tensor& dst,
6 + const std::map<int64_t, int64_t>& block_mapping);
7 +
8 +void copy_cache_blocks(
9 + torch::Tensor& src,
10 + torch::Tensor& dst,
11 + const std::map<int64_t, int64_t>& block_mapping) {
12 + copy_blocks(src, dst, block_mapping);
13 +}
14 +
15 +PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
16 + m.def(
17 + "copy_cache_blocks",
18 + &copy_cache_blocks,
19 + "Copy the cache blocks from src to dst");
20 +}
csrc/cache_kernel.cu new
+43
@@ -0,0 +1,43 @@
1 +#include <torch/extension.h>
2 +
3 +#include <ATen/cuda/CUDAContext.h>
4 +
5 +#include <cassert>
6 +#include <map>
7 +
8 +void copy_blocks(
9 + torch::Tensor& src,
10 + torch::Tensor& dst,
11 + const std::map<int64_t, int64_t>& block_mapping) {
12 + torch::Device src_device = src.device();
13 + torch::Device dst_device = dst.device();
14 + cudaMemcpyKind memcpy_type;
15 + if (src_device.is_cuda() && dst_device.is_cuda()) {
16 + assert(src_device.index() == dst_device.index());
17 + memcpy_type = cudaMemcpyDeviceToDevice;
18 + } else if (src_device.is_cuda() && dst_device.is_cpu()) {
19 + memcpy_type = cudaMemcpyDeviceToHost;
20 + } else if (src_device.is_cpu() && dst_device.is_cuda()) {
21 + memcpy_type = cudaMemcpyHostToDevice;
22 + } else {
23 + assert(false);
24 + }
25 +
26 + void *src_ptr = src.data_ptr();
27 + void *dst_ptr = dst.data_ptr();
28 +
29 + const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
30 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
31 + for (const auto& pair : block_mapping) {
32 + int64_t src_block_number = pair.first;
33 + int64_t dst_block_number = pair.second;
34 + int64_t src_offset = src_block_number * block_size_in_bytes;
35 + int64_t dst_offset = dst_block_number * block_size_in_bytes;
36 + cudaMemcpyAsync(
37 + dst_ptr + dst_offset,
38 + src_ptr + src_offset,
39 + block_size_in_bytes,
40 + memcpy_type,
41 + stream);
42 + }
43 +}
setup.py
+23
@@ -0,0 +1,23 @@
1 +import setuptools
2 +from torch.utils import cpp_extension
3 +
4 +CXX_FLAGS = ['-g']
5 +NVCC_FLAGS = ['-O2']
6 +
7 +
8 +ext_modules = []
9 +
10 +# Cache operations.
11 +cache_extension = cpp_extension.CUDAExtension(
12 + name='cacheflow.ops',
13 + sources=['csrc/cache.cpp', 'csrc/cache_kernel.cu'],
14 + extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
15 +)
16 +ext_modules.append(cache_extension)
17 +
18 +setuptools.setup(
19 + name='cacheflow',
20 + requires_python='>=3.9',
21 + ext_modules=ext_modules,
22 + cmdclass={'build_ext': cpp_extension.BuildExtension},
23 +)