@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0f40557af6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Implement block copy kernel to optimize beam search (#32)

Woosuk Kwon committed Apr 7, 2023 at 17:45 UTC 0f40557af6141ced118b81f2a04e651a0c6c9dbd
6 files changed +155 -49
benchmark/benchmark_latency.py
+6 -3
@@ -50,14 +50,15 @@ def main(args: argparse.Namespace):
50 block_size=args.block_size,
51 )
52 sampling_params_dict = {
53 - 'n': 1,
54 - 'temperature': 0.0,
53 + 'n': args.n,
54 + 'temperature': 0.0 if args.use_beam_search else 1.0,
55 'top_p': 1.0,
56 - 'use_beam_search': False,
56 + 'use_beam_search': args.use_beam_search,
57 'stop_token_ids': set(),
58 'max_num_steps': args.output_len,
59 }
60 sampling_params = SamplingParams.from_dict(sampling_params_dict)
61 + print(sampling_params)
62 input_token_ids = [0] * args.input_len
63
64 def profile_step(profile=False):
@@ -93,6 +94,8 @@ if __name__ == '__main__':
94 parser.add_argument('--input-len', type=int, default=32)
95 parser.add_argument('--output-len', type=int, default=128)
96 parser.add_argument('--batch-size', type=int, default=8)
97 + parser.add_argument('--n', type=int, default=1)
98 + parser.add_argument('--use-beam-search', action='store_true')
99 args = parser.parse_args()
100 args.max_num_batched_tokens = max(
101 args.max_num_batched_tokens, args.batch_size * args.input_len)
cacheflow/models/sample.py
+3 -2
@@ -185,9 +185,10 @@ def _sample_from_generation_tokens(
185 vocab_size = logprobs.size(-1)
186 beam_width = len(seq_ids)
187 _, topk_ids = torch.topk(logprobs.flatten(), beam_width)
188 - seq_idx = torch.div(topk_ids, vocab_size, rounding_mode='floor').tolist()
188 + topk_ids = topk_ids.tolist()
189 + seq_idx = [i // vocab_size for i in topk_ids]
190 beam_seq_ids = [seq_ids[i] for i in seq_idx]
190 - token_ids = (topk_ids % vocab_size).tolist()
191 + token_ids = [i % vocab_size for i in topk_ids]
192
193 beam_outputs: Dict[int, Tuple[int, int]] = {}
194 outstanding_beams: List[Tuple[int, int]] = []
cacheflow/worker/cache_engine.py
+4 -20
@@ -120,24 +120,8 @@ class CacheEngine:
120 def swap_out(self, src_to_dst: Dict[int, int]) -> None:
121 self._swap(self.gpu_cache, self.cpu_cache, src_to_dst)
122
123 - def _copy(
124 - self,
125 - src: List[KVCache],
126 - dst: List[KVCache],
127 - src_to_dsts: Dict[int, List[int]],
128 - ) -> None:
129 - with torch.cuda.stream(self.cache_stream):
130 - for i in range(self.num_layers):
131 - src_key_cache, src_value_cache = src[i]
132 - dst_key_cache, dst_value_cache = dst[i]
133 - # Copy the key blocks.
134 - cache_ops.copy_blocks(
135 - src_key_cache, dst_key_cache, src_to_dsts)
136 - # Copy the value blocks.
137 - cache_ops.copy_blocks(
138 - src_value_cache, dst_value_cache, src_to_dsts)
139 - event = self.events[i]
140 - event.record(stream=self.cache_stream)
141 -
123 def copy(self, src_to_dsts: Dict[int, List[int]]) -> None:
143 - self._copy(self.gpu_cache, self.gpu_cache, src_to_dsts)
124 + key_caches = [key_cache for key_cache, _ in self.gpu_cache]
125 + value_caches = [value_cache for _, value_cache in self.gpu_cache]
126 + # NOTE(woosuk): This operation implicitly synchronizes the CPU and GPU.
127 + cache_ops.copy_blocks(key_caches, value_caches, src_to_dsts)
csrc/cache.cpp
+2 -2
@@ -9,8 +9,8 @@ void swap_blocks(
9 const std::map<int64_t, int64_t>& block_mapping);
10
11 void copy_blocks(
12 - torch::Tensor& src,
13 - torch::Tensor& dst,
12 + std::vector<torch::Tensor>& key_caches,
13 + std::vector<torch::Tensor>& value_caches,
14 const std::map<int64_t, std::vector<int64_t>>& block_mapping);
15
16 void reshape_and_cache(
csrc/cache_kernels.cu
+82 -22
@@ -43,33 +43,93 @@ void swap_blocks(
43 }
44 }
45
46 -void copy_blocks(
47 - torch::Tensor& src,
48 - torch::Tensor& dst,
49 - const std::map<int64_t, std::vector<int64_t>>& block_mapping) {
50 - torch::Device src_device = src.device();
51 - torch::Device dst_device = dst.device();
52 - assert(src_device.is_cuda() && dst_device.is_cuda());
53 - cudaMemcpyKind memcpy_type = cudaMemcpyDeviceToDevice;
46 +namespace cacheflow {
47
55 - void *src_ptr = src.data_ptr();
56 - void *dst_ptr = dst.data_ptr();
48 +// Grid: (num_layers, num_pairs)
49 +template<typename scalar_t>
50 +__global__ void copy_blocks_kernel(
51 + int64_t* key_cache_ptrs,
52 + int64_t* value_cache_ptrs,
53 + const int* __restrict__ block_mapping,
54 + const int numel_per_block) {
55 + const int layer_idx = blockIdx.x;
56 + const int pair_idx = blockIdx.y;
57 +
58 + scalar_t* key_cache = reinterpret_cast<scalar_t*>(key_cache_ptrs[layer_idx]);
59 + scalar_t* value_cache = reinterpret_cast<scalar_t*>(value_cache_ptrs[layer_idx]);
60 + int src_block_number = block_mapping[2 * pair_idx];
61 + int dst_block_number = block_mapping[2 * pair_idx + 1];
62 +
63 + const int src_block_offset = src_block_number * numel_per_block;
64 + const int dst_block_offset = dst_block_number * numel_per_block;
65 + for (int i = threadIdx.x; i < numel_per_block; i += blockDim.x) {
66 + int src_offset = src_block_offset + i;
67 + int dst_offset = dst_block_offset + i;
68 + key_cache[dst_offset] = key_cache[src_offset];
69 + }
70 + for (int i = threadIdx.x; i < numel_per_block; i += blockDim.x) {
71 + int src_offset = src_block_offset + i;
72 + int dst_offset = dst_block_offset + i;
73 + value_cache[dst_offset] = value_cache[src_offset];
74 + }
75 +}
76
58 - const int64_t block_size_in_bytes = src.element_size() * src[0].numel();
59 - const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
77 +} // namespace cacheflow
78 +
79 +void copy_blocks(
80 + std::vector<torch::Tensor>& key_caches,
81 + std::vector<torch::Tensor>& value_caches,
82 + const std::map<int64_t, std::vector<int64_t>>& block_mapping) {
83 + int num_layers = key_caches.size();
84 + TORCH_CHECK(num_layers == value_caches.size());
85 + if (num_layers == 0) {
86 + return;
87 + }
88 + torch::Device cache_device = key_caches[0].device();
89 + TORCH_CHECK(cache_device.is_cuda());
90 +
91 + // Create data structures for the kernel.
92 + // Create an array of pointers to the key and value caches.
93 + int64_t key_cache_ptrs[num_layers];
94 + int64_t value_cache_ptrs[num_layers];
95 + for (int layer_idx = 0; layer_idx < num_layers; ++layer_idx) {
96 + key_cache_ptrs[layer_idx] = reinterpret_cast<int64_t>(key_caches[layer_idx].data_ptr());
97 + value_cache_ptrs[layer_idx] = reinterpret_cast<int64_t>(value_caches[layer_idx].data_ptr());
98 + }
99 + // Create block mapping array.
100 + std::vector<int> block_mapping_vec;
101 for (const auto& pair : block_mapping) {
61 - int64_t src_block_number = pair.first;
62 - for (int64_t dst_block_number : pair.second) {
63 - int64_t src_offset = src_block_number * block_size_in_bytes;
64 - int64_t dst_offset = dst_block_number * block_size_in_bytes;
65 - cudaMemcpyAsync(
66 - dst_ptr + dst_offset,
67 - src_ptr + src_offset,
68 - block_size_in_bytes,
69 - memcpy_type,
70 - stream);
102 + int src_block_number = pair.first;
103 + for (int dst_block_number : pair.second) {
104 + block_mapping_vec.push_back(src_block_number);
105 + block_mapping_vec.push_back(dst_block_number);
106 }
107 }
108 + int* block_mapping_array = block_mapping_vec.data();
109 + int num_pairs = block_mapping_vec.size() / 2;
110 +
111 + // Move the data structures to the GPU.
112 + // NOTE: This synchronizes the CPU and GPU.
113 + torch::Tensor key_cache_ptrs_tensor = torch::from_blob(
114 + key_cache_ptrs, {num_layers}, torch::kInt64).to(cache_device);
115 + torch::Tensor value_cache_ptrs_tensor = torch::from_blob(
116 + value_cache_ptrs, {num_layers}, torch::kInt64).to(cache_device);
117 + torch::Tensor block_mapping_tensor = torch::from_blob(
118 + block_mapping_array, {2 * num_pairs}, torch::kInt).to(cache_device);
119 +
120 + // Launch the kernel.
121 + const int numel_per_block = key_caches[0][0].numel();
122 + dim3 grid(num_layers, num_pairs);
123 + dim3 block(std::min(1024, numel_per_block));
124 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
125 + AT_DISPATCH_FLOATING_TYPES_AND_HALF(
126 + key_caches[0].scalar_type(), "copy_blocks_kernel", ([&] {
127 + cacheflow::copy_blocks_kernel<scalar_t><<<grid, block, 0, stream>>>(
128 + key_cache_ptrs_tensor.data_ptr<int64_t>(),
129 + value_cache_ptrs_tensor.data_ptr<int64_t>(),
130 + block_mapping_tensor.data_ptr<int>(),
131 + numel_per_block);
132 + }));
133 }
134
135 namespace cacheflow {
tests/kernels/cache.py
+58
@@ -5,6 +5,61 @@ import torch
5 from cacheflow import cache_ops
6
7
8 +def test_copy_blocks(
9 + num_mappings: int,
10 + num_layers: int,
11 + num_heads: int,
12 + head_size: int,
13 + block_size: int,
14 + num_blocks: int,
15 + dtype: torch.dtype,
16 +) -> None:
17 + # Generate random block mappings.
18 + src_blocks = random.sample(range(num_blocks), num_mappings)
19 + remainig_blocks = list(set(range(num_blocks)) - set(src_blocks))
20 + dst_blocks = random.sample(remainig_blocks, num_mappings)
21 + block_mapping = {src: [dst] for src, dst in zip(src_blocks, dst_blocks)}
22 +
23 + # Create the KV cache.
24 + x = 16 // torch.tensor([], dtype=dtype).element_size()
25 + key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
26 + key_caches = []
27 + for _ in range(num_layers):
28 + key_cache = torch.randn(
29 + size=key_cache_shape, dtype=dtype, device='cuda')
30 + key_caches.append(key_cache)
31 + cloned_key_caches = []
32 + for key_cache in key_caches:
33 + cloned_key_caches.append(key_cache.clone())
34 +
35 + value_cache_shape = (num_blocks, num_heads, head_size, block_size)
36 + value_caches = []
37 + for _ in range(num_layers):
38 + value_cache = torch.randn(
39 + size=value_cache_shape, dtype=dtype, device='cuda')
40 + value_caches.append(value_cache)
41 + cloned_value_caches = []
42 + for value_cache in value_caches:
43 + cloned_value_caches.append(value_cache.clone())
44 +
45 + # Call the copy blocks kernel.
46 + cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
47 +
48 + # Reference implementation.
49 + for src, dsts in block_mapping.items():
50 + for dst in dsts:
51 + for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
52 + cloned_key_cache[dst] = cloned_key_cache[src]
53 + for value_cache, cloned_value_cache in zip(value_caches, cloned_value_caches):
54 + cloned_value_cache[dst] = cloned_value_cache[src]
55 +
56 + # Compare the results.
57 + for key_cache, cloned_key_cache in zip(key_caches, cloned_key_caches):
58 + assert torch.allclose(key_cache, cloned_key_cache)
59 + for value_cache, cloned_value_cache in zip(value_caches, cloned_value_caches):
60 + assert torch.allclose(value_cache, cloned_value_cache)
61 +
62 +
63 def test_reshape_and_cache(
64 num_tokens: int,
65 num_heads: int,
@@ -46,6 +101,9 @@ def test_reshape_and_cache(
101
102 @torch.inference_mode()
103 def test_cache() -> None:
104 + test_copy_blocks(
105 + num_mappings=23, num_layers=7, num_heads=17, head_size=16,
106 + block_size=8, num_blocks=1024, dtype=torch.half)
107 test_reshape_and_cache(
108 num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
109 dtype=torch.half)