A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Implement block copy kernel to optimize beam search (#32)
Woosuk Kwon committed
Apr 7, 2023 at 17:45 UTC
0f40557af6141ced118b81f2a04e651a0c6c9dbd
6 files changed
+155
-49
benchmark/benchmark_latency.py
+6
-3
@@ -50,14 +50,15 @@ def main(args: argparse.Namespace):
50
block_size=args.block_size,
51
)
52
sampling_params_dict = {
53
- 'n': 1,
54
- 'temperature': 0.0,
53
+ 'n': args.n,
54
+ 'temperature': 0.0 if args.use_beam_search else 1.0,
55
'top_p': 1.0,
56
- 'use_beam_search': False,
56
+ 'use_beam_search': args.use_beam_search,
57
'stop_token_ids': set(),
58
'max_num_steps': args.output_len,
59
}
60
sampling_params = SamplingParams.from_dict(sampling_params_dict)
61
+ print(sampling_params)
62
input_token_ids = [0] * args.input_len
63
64
def profile_step(profile=False):
@@ -93,6 +94,8 @@ if __name__ == '__main__':
94
parser.add_argument('--input-len', type=int, default=32)
95
parser.add_argument('--output-len', type=int, default=128)
96
parser.add_argument('--batch-size', type=int, default=8)
97
+ parser.add_argument('--n', type=int, default=1)
98
+ parser.add_argument('--use-beam-search', action='store_true')
99
args = parser.parse_args()
100
args.max_num_batched_tokens = max(
101
args.max_num_batched_tokens, args.batch_size * args.input_len)
cacheflow/models/sample.py
+3
-2
@@ -185,9 +185,10 @@ def _sample_from_generation_tokens(
185
vocab_size = logprobs.size(-1)
186
beam_width = len(seq_ids)
187
_, topk_ids = torch.topk(logprobs.flatten(), beam_width)
188
- seq_idx = torch.div(topk_ids, vocab_size, rounding_mode='floor').tolist()
188
+ topk_ids = topk_ids.tolist()
189
+ seq_idx = [i // vocab_size for i in topk_ids]
190
beam_seq_ids = [seq_ids[i] for i in seq_idx]
190
- token_ids = (topk_ids % vocab_size).tolist()
191
+ token_ids = [i % vocab_size for i in topk_ids]
192
193
beam_outputs: Dict[int, Tuple[int, int]] = {}
194
outstanding_beams: List[Tuple[int, int]] = []
cacheflow/worker/cache_engine.py
+4
-20
@@ -120,24 +120,8 @@ class CacheEngine:
120
def swap_out(self, src_to_dst: Dict[int, int]) -> None:
121
self._swap(self.gpu_cache, self.cpu_cache, src_to_dst)
122
123
- def _copy(
124
- self,
125
- src: List[KVCache],
126
- dst: List[KVCache],
127
- src_to_dsts: Dict[int, List[int]],
128
- ) -> None:
129
- with torch.cuda.stream(self.cache_stream):
130
- for i in range(self.num_layers):
131
- src_key_cache, src_value_cache = src[i]
132
- dst_key_cache, dst_value_cache = dst[i]
133
- # Copy the key blocks.
134
- cache_ops.copy_blocks(
135
- src_key_cache, dst_key_cache, src_to_dsts)
136
- # Copy the value blocks.
137
- cache_ops.copy_blocks(
138
- src_value_cache, dst_value_cache, src_to_dsts)
139
- event = self.events[i]
140
- event.record(stream=self.cache_stream)
141
-
123
def copy(self, src_to_dsts: Dict[int, List[int]]) -> None:
143
- self._copy(self.gpu_cache, self.gpu_cache, src_to_dsts)
124
+ key_caches = [key_cache for key_cache, _ in self.gpu_cache]
125
+ value_caches = [value_cache for _, value_cache in self.gpu_cache]
126
+ # NOTE(woosuk): This operation implicitly synchronizes the CPU and GPU.
127
+ cache_ops.copy_blocks(key_caches, value_caches, src_to_dsts)
csrc/cache.cpp
+2
-2
@@ -9,8 +9,8 @@ void swap_blocks(
9
const std::map<int64_t, int64_t>& block_mapping);
10
11
void copy_blocks(
12
- torch::Tensor& src,
13
- torch::Tensor& dst,
12
+ std::vector<torch::Tensor>& key_caches,
13
+ std::vector<torch::Tensor>& value_caches,
14
const std::map<int64_t, std::vector<int64_t>>& block_mapping);
15
16
void reshape_and_cache(
csrc/cache_kernels.cu
+82
-22
@@ -43,33 +43,93 @@ void swap_blocks(
43
}
44
}
45
46
-void copy_blocks(
47
- torch::Tensor& src,
48
- torch::Tensor& dst,
49
- const std::map<int64_t, std::vector<int64_t>>& block_mapping) {
50
- torch::Device src_device = src.device();
51
- torch::Device dst_device = dst.device();
52
- assert(src_device.is_cuda() && dst_device.is_cuda());
53
- cudaMemcpyKind memcpy_type = cudaMemcpyDeviceToDevice;
46
+namespace cacheflow {
47
55
- void *src_ptr = src.data_ptr();
56
- void *dst_ptr = dst.data_ptr();
48
+// Grid: (num_layers, num_pairs)
49
+template<typename scalar_t>
50
+__global__ void copy_blocks_kernel(
51
+ int64_t* key_cache_ptrs,
52
+ int64_t* value_cache_ptrs,
53
+ const int* __restrict__ block_mapping,
54
+ const int numel_per_block) {
55
+ const int layer_idx = blockIdx.x;
56
+ const int pair_idx = blockIdx.y;
57
+
58
+ scalar_t* key_cache = reinterpret_cast<scalar_t*>(key_cache_ptrs[layer_idx]);
59
+ scalar_t* value_cache = reinterpret_cast<scalar_t*>(value_cache_ptrs[layer_idx]);
60
+ int src_block_number = block_mapping[2 * pair_idx];
61
+ int dst_block_number = block_mapping[2 * pair_idx + 1];
62
+
63
+ const int src_block_offset = src_block_number * numel_per_block;
64
+ const int dst_block_offset = dst_block_number * numel_per_block;
65
+ for (int i = threadIdx.x; i < numel_per_block; i += blockDim.x) {
66
+ int src_offset = src_block_offset + i;
67
+ int dst_offset = dst_block_offset + i;
68
+ key_cache[dst_offset] = key_cache[src_offset];
69
+ }
70
+ for (int i = threadIdx.x; i < numel_per_block; i += blockDim.x) {
71
+ int src_offset = src_block_offset + i;
72
+ int dst_offset = dst_block_offset + i;
73
+ value_cache[dst_offset] = value_cache[src_offset];
74
+ }
75
+}
76
58
- const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
59
- const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
77
+} // namespace cacheflow
78
+
79
+void copy_blocks(
80
+ std::vector<torch::Tensor>& key_caches,
81
+ std::vector<torch::Tensor>& value_caches,
82
+ const std::map<int64_t, std::vector<int64_t>>& block_mapping) {
83
+ int num_layers = key_caches.size();
84
+ TORCH_CHECK(num_layers == value_caches.size());
85
+ if (num_layers == 0) {
86
+ return;
87
+ }
88
+ torch::Device cache_device = key_caches[0].device();
89
+ TORCH_CHECK(cache_device.is_cuda());
90
+
91
+ // Create data structures for the kernel.
92
+ // Create an array of pointers to the key and value caches.
93
+ int64_t key_cache_ptrs[num_layers];
94
+ int64_t value_cache_ptrs[num_layers];
95
+ for (int layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
96
+ key_cache_ptrs[layer_idx] = reinterpret_cast<int64_t>(key_caches[layer_idx].data_ptr());
97
+ value_cache_ptrs[layer_idx] = reinterpret_cast<int64_t>(value_caches[layer_idx].data_ptr());
98
+ }
99
+ // Create block mapping array.
100
+ std::vector<int> block_mapping_vec;
101
for (const auto& pair : block_mapping) {
61
- int64_t src_block_number = pair.first;
62
- for (int64_t dst_block_number : pair.second) {
63
- int64_t src_offset = src_block_number * block_size_in_bytes;
64
- int64_t dst_offset = dst_block_number * block_size_in_bytes;
65
- cudaMemcpyAsync(
66
- dst_ptr + dst_offset,
67
- src_ptr + src_offset,
68
- block_size_in_bytes,
69
- memcpy_type,
70
- stream);
102
+ int src_block_number = pair.first;
103
+ for (int dst_block_number : pair.second) {
104
+ block_mapping_vec.push_back(src_block_number);
105
+ block_mapping_vec.push_back(dst_block_number);
106
}
107
}
108
+ int* block_mapping_array = block_mapping_vec.data();
109
+ int num_pairs = block_mapping_vec.size() / 2;
110
+
111
+ // Move the data structures to the GPU.
112
+ // NOTE: This synchronizes the CPU and GPU.
113
+ torch::Tensor key_cache_ptrs_tensor = torch::from_blob(
114
+ key_cache_ptrs, {num_layers}, torch::kInt64).to(cache_device);
115
+ torch::Tensor value_cache_ptrs_tensor = torch::from_blob(
116
+ value_cache_ptrs, {num_layers}, torch::kInt64).to(cache_device);
117
+ torch::Tensor block_mapping_tensor = torch::from_blob(
118
+ block_mapping_array, {2 * num_pairs}, torch::kInt).to(cache_device);
119
+
120
+ // Launch the kernel.
121
+ const int numel_per_block = key_caches[0][0].numel();
122
+ dim3 grid(num_layers, num_pairs);
123
+ dim3 block(std::min(1024, numel_per_block));
124
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
125
+ AT_DISPATCH_FLOATING_TYPES_AND_HALF(
126
+ key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
127
+ cacheflow::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
128
+ key_cache_ptrs_tensor.data_ptr<int64_t>(),
129
+ value_cache_ptrs_tensor.data_ptr<int64_t>(),
130
+ block_mapping_tensor.data_ptr<int>(),
131
+ numel_per_block);
132
+ }));
133
}
134
135
namespace cacheflow {
tests/kernels/cache.py
+58
@@ -5,6 +5,61 @@ import torch
5
from cacheflow import cache_ops
6
7
8
+def test_copy_blocks(
9
+ num_mappings: int,
10
+ num_layers: int,
11
+ num_heads: int,
12
+ head_size: int,
13
+ block_size: int,
14
+ num_blocks: int,
15
+ dtype: torch.dtype,
16
+) -> None:
17
+ # Generate random block mappings.
18
+ src_blocks = random.sample(range(num_blocks), num_mappings)
19
+ remainig_blocks = list(set(range(num_blocks)) - set(src_blocks))
20
+ dst_blocks = random.sample(remainig_blocks, num_mappings)
21
+ block_mapping = {src: [dst] for src, dst in zip(src_blocks, dst_blocks)}
22
+
23
+ # Create the KV cache.
24
+ x = 16 // torch.tensor([], dtype=dtype).element_size()
25
+ key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
26
+ key_caches = []
27
+ for _ in range(num_layers):
28
+ key_cache = torch.randn(
29
+ size=key_cache_shape, dtype=dtype, device='cuda')
30
+ key_caches.append(key_cache)
31
+ cloned_key_caches = []
32
+ for key_cache in key_caches:
33
+ cloned_key_caches.append(key_cache.clone())
34
+
35
+ value_cache_shape = (num_blocks, num_heads, head_size, block_size)
36
+ value_caches = []
37
+ for _ in range(num_layers):
38
+ value_cache = torch.randn(
39
+ size=value_cache_shape, dtype=dtype, device='cuda')
40
+ value_caches.append(value_cache)
41
+ cloned_value_caches = []
42
+ for value_cache in value_caches:
43
+ cloned_value_caches.append(value_cache.clone())
44
+
45
+ # Call the copy blocks kernel.
46
+ cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
47
+
48
+ # Reference implementation.
49
+ for src, dsts in block_mapping.items():
50
+ for dst in dsts:
51
+ for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
52
+ cloned_key_cache[dst] = cloned_key_cache[src]
53
+ for value_cache, cloned_value_cache in zip(value_caches, cloned_value_caches):
54
+ cloned_value_cache[dst] = cloned_value_cache[src]
55
+
56
+ # Compare the results.
57
+ for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
58
+ assert torch.allclose(key_cache, cloned_key_cache)
59
+ for value_cache, cloned_value_cache in zip(value_caches, cloned_value_caches):
60
+ assert torch.allclose(value_cache, cloned_value_cache)
61
+
62
+
63
def test_reshape_and_cache(
64
num_tokens: int,
65
num_heads: int,
@@ -46,6 +101,9 @@ def test_reshape_and_cache(
101
102
@torch.inference_mode()
103
def test_cache() -> None:
104
+ test_copy_blocks(
105
+ num_mappings=23, num_layers=7, num_heads=17, head_size=16,
106
+ block_size=8, num_blocks=1024, dtype=torch.half)
107
test_reshape_and_cache(
108
num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
109
dtype=torch.half)