@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e3cec88aa5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Memcpy kernel for flash attention (#29)

* optimize * add benchmark * add assert * add test

Siyuan (Ryans) Zhuang committed Apr 10, 2023 at 18:22 UTC e3cec88aa5b7ac391e4aa6dc9b6388100d59d8f9
4 files changed +293
benchmark/benchmark_cache.py new
+81
@@ -0,0 +1,81 @@
1 +import functools
2 +import random
3 +import time
4 +
5 +import torch
6 +
7 +from cacheflow import cache_ops
8 +
9 +
10 +def benchmark(name, f, size: int, num_warmup = 10, num_iters = 100):
11 + for _ in range(num_warmup):
12 + f()
13 + torch.cuda.synchronize()
14 +
15 + start = time.time()
16 + for _ in range(num_iters):
17 + f()
18 + torch.cuda.synchronize()
19 + end = time.time()
20 + avg_time = (end - start) / num_iters
21 + print(f'[Latency] {name}: {avg_time * 1000:.3f} ms')
22 + print(f'[Throughput] {name}: {size / avg_time / 2 ** 30:.3f} GB/s')
23 +
24 +
25 +@torch.inference_mode()
26 +def test_gather_cached_kv(
27 + num_tokens: int,
28 + num_heads: int,
29 + head_size: int,
30 + block_size: int,
31 + num_blocks: int,
32 + dtype: torch.dtype,
33 +) -> None:
34 + print(f'num_tokens: {num_tokens}, num_heads: {num_heads}, '
35 + f'head_size: {head_size}, block_size: {block_size}, '
36 + f'num_blocks: {num_blocks}, dtype: {dtype}')
37 +
38 + num_slots = block_size * num_blocks
39 + slot_mapping = random.sample(range(num_slots), num_tokens)
40 + slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
41 +
42 + qkv = torch.randn(
43 + num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
44 + _, key, value = qkv.unbind(dim=1)
45 +
46 + x = 16 // torch.tensor([], dtype=dtype).element_size()
47 + key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
48 + key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
49 +
50 + value_cache_shape = (num_blocks, num_heads, head_size, block_size)
51 + value_cache = torch.randn(
52 + size=value_cache_shape, dtype=dtype, device='cuda')
53 +
54 + # Run Flash attention.
55 + def run():
56 + cache_ops.gather_cached_kv(key, value, key_cache, value_cache, slot_mapping)
57 +
58 + benchmark('gather_cached_kv', run,
59 + size=num_tokens * num_heads * head_size * 2 * qkv.element_size())
60 +
61 +
62 +if __name__ == '__main__':
63 + BLOCK_SIZE = 8
64 + NUM_BLOCKS = 1024
65 + DTYPE = torch.half
66 +
67 + # LLaMA-13B and OPT-13B
68 + NUM_HEADS = 40
69 + HEAD_SIZE = 128
70 +
71 + run_benchmark = functools.partial(
72 + test_gather_cached_kv,
73 + num_heads=NUM_HEADS,
74 + head_size=HEAD_SIZE,
75 + block_size=BLOCK_SIZE,
76 + num_blocks=NUM_BLOCKS,
77 + dtype=DTYPE,
78 + )
79 +
80 + for i in range(6, 12):
81 + run_benchmark(num_tokens=2 ** i)
csrc/cache.cpp
+11
@@ -20,6 +20,13 @@ void reshape_and_cache(
20 torch::Tensor& value_cache,
21 torch::Tensor& slot_mapping);
22
23 +void gather_cached_kv(
24 + torch::Tensor& key,
25 + torch::Tensor& value,
26 + torch::Tensor& key_cache,
27 + torch::Tensor& value_cache,
28 + torch::Tensor& slot_mapping);
29 +
30 PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
31 m.def(
32 "swap_blocks",
@@ -33,4 +40,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
40 "reshape_and_cache",
41 &reshape_and_cache,
42 "Reshape the key and value tensors and cache them");
43 + m.def(
44 + "gather_cached_kv",
45 + &gather_cached_kv,
46 + "Gather key and value from the cache into contiguous QKV tensors");
47 }
csrc/cache_kernels.cu
+157
@@ -176,6 +176,124 @@ __global__ void reshape_and_cache_kernel(
176 }
177 }
178
179 +// Grid: (num_blocks, block_size).
180 +template<typename scalar_t>
181 +__global__ void gather_cached_kv_kernel(
182 + scalar_t* __restrict__ key, // [num_tokens, [stride], num_heads, head_size]
183 + scalar_t* __restrict__ value, // [num_tokens, [stride], num_heads, head_size]
184 + const scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
185 + const scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
186 + const int* __restrict__ slot_mapping, // [num_tokens]
187 + const int key_stride,
188 + const int value_stride,
189 + const int num_heads,
190 + const int head_size,
191 + const int block_size,
192 + const int x) {
193 + const int token_idx = blockIdx.x;
194 + const int slot_idx = slot_mapping[token_idx];
195 + const int block_idx = slot_idx / block_size;
196 + const int block_offset = slot_idx % block_size;
197 +
198 + const int num_tokens = num_heads * head_size;
199 + for (int i = threadIdx.x; i < num_tokens; i += blockDim.x) {
200 + const int tgt_key_idx = token_idx * key_stride + i;
201 + const int tgt_value_idx = token_idx * value_stride + i;
202 +
203 + const int head_idx = i / head_size;
204 + const int head_offset = i % head_size;
205 + const int x_idx = head_offset / x; // the offset of the [head_size/x] dimension
206 + const int x_offset = head_offset % x;
207 +
208 + const int src_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
209 + + head_idx * (head_size / x) * block_size * x
210 + + x_idx * block_size * x
211 + + block_offset * x
212 + + x_offset;
213 + const int src_value_idx = block_idx * num_heads * head_size * block_size
214 + + head_idx * head_size * block_size
215 + + head_offset * block_size
216 + + block_offset;
217 +
218 + key[tgt_key_idx] = __ldg(&key_cache[src_key_idx]);
219 + value[tgt_value_idx] = __ldg(&value_cache[src_value_idx]);
220 + }
221 +}
222 +
223 +template <typename scalar_t>
224 +__global__ void gather_cached_kv_kernel_optimized(
225 + scalar_t *__restrict__ key, // [num_tokens, [stride], num_heads, head_size]
226 + scalar_t *__restrict__ value, // [num_tokens, [stride], num_heads, head_size]
227 + const scalar_t *__restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
228 + const scalar_t *__restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
229 + const int *__restrict__ slot_mapping, // [num_tokens]
230 + const int key_stride,
231 + const int value_stride,
232 + const int num_heads,
233 + const int head_size,
234 + const int block_size,
235 + const int x)
236 +{
237 + const int token_idx = blockIdx.x;
238 + const int slot_idx = slot_mapping[token_idx];
239 + const int block_idx = slot_idx / block_size;
240 + const int block_offset = slot_idx % block_size;
241 +
242 + const int dim = num_heads * head_size;
243 + assert(dim % 4 == 0); // this is true for known use cases
244 + const int unroll_factor = 4;
245 + const int unrolled_dim = dim / unroll_factor;
246 +
247 + for (int i = threadIdx.x; i < unrolled_dim; i += blockDim.x)
248 + {
249 + int tgt_key_indices[unroll_factor];
250 + int tgt_value_indices[unroll_factor];
251 + int src_key_indices[unroll_factor];
252 + int src_value_indices[unroll_factor];
253 + scalar_t keys_to_store[unroll_factor];
254 + scalar_t values_to_store[unroll_factor];
255 +
256 + #pragma unroll
257 + for (int j = 0; j < unroll_factor; ++j)
258 + {
259 + int index = i + j * unrolled_dim;
260 +
261 + const int tgt_key_idx = token_idx * key_stride + index;
262 + const int tgt_value_idx = token_idx * value_stride + index;
263 +
264 + const int head_idx = index / head_size;
265 + const int head_offset = index % head_size;
266 + const int x_idx = head_offset / x;
267 + const int x_offset = head_offset % x;
268 +
269 + const int src_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
270 + + head_idx * (head_size / x) * block_size * x
271 + + x_idx * block_size * x
272 + + block_offset * x
273 + + x_offset;
274 + const int src_value_idx = block_idx * num_heads * head_size * block_size
275 + + head_idx * head_size * block_size
276 + + head_offset * block_size
277 + + block_offset;
278 +
279 + tgt_key_indices[j] = tgt_key_idx;
280 + tgt_value_indices[j] = tgt_value_idx;
281 + src_key_indices[j] = src_key_idx;
282 + src_value_indices[j] = src_value_idx;
283 +
284 + keys_to_store[j] = __ldg(&key_cache[src_key_idx]);
285 + values_to_store[j] = __ldg(&value_cache[src_value_idx]);
286 + }
287 +
288 + #pragma unroll
289 + for (int j = 0; j < unroll_factor; ++j)
290 + {
291 + key[tgt_key_indices[j]] = keys_to_store[j];
292 + value[tgt_value_indices[j]] = values_to_store[j];
293 + }
294 + }
295 +}
296 +
297 } // namespace cacheflow
298
299 void reshape_and_cache(
@@ -215,3 +333,42 @@ void reshape_and_cache(
333 x);
334 });
335 }
336 +
337 +
338 +void gather_cached_kv(
339 + torch::Tensor& key, // [out] [num_tokens, num_heads, head_size]
340 + torch::Tensor& value, // [out] [num_tokens, num_heads, head_size]
341 + torch::Tensor& key_cache, // [in] [num_blocks, num_heads, head_size/x, block_size, x]
342 + torch::Tensor& value_cache, // [in] [num_blocks, num_heads, head_size, block_size]
343 + torch::Tensor& slot_mapping) // [in] [num_tokens]
344 +{
345 + int num_tokens = key.size(0);
346 + int num_heads = key.size(1);
347 + int head_size = key.size(2);
348 + int block_size = key_cache.size(3);
349 + int x = key_cache.size(4);
350 +
351 + int key_stride = key.stride(0);
352 + int value_stride = value.stride(0);
353 +
354 + dim3 grid(num_tokens);
355 + dim3 block(std::min(num_heads * head_size, 512));
356 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
357 + AT_DISPATCH_FLOATING_TYPES_AND_HALF(
358 + key.scalar_type(),
359 + "gather_cached_kv_kernel_optimized",
360 + [&] {
361 + cacheflow::gather_cached_kv_kernel_optimized<scalar_t><<<grid, block, 0, stream>>>(
362 + key.data_ptr<scalar_t>(),
363 + value.data_ptr<scalar_t>(),
364 + key_cache.data_ptr<scalar_t>(),
365 + value_cache.data_ptr<scalar_t>(),
366 + slot_mapping.data_ptr<int>(),
367 + key_stride,
368 + value_stride,
369 + num_heads,
370 + head_size,
371 + block_size,
372 + x);
373 + });
374 +}
tests/kernels/cache.py
+44
@@ -99,6 +99,47 @@ def test_reshape_and_cache(
99 assert torch.allclose(value_cache, cloned_value_cache)
100
101
102 +def test_gather_cached_kv(
103 + num_tokens: int,
104 + num_heads: int,
105 + head_size: int,
106 + block_size: int,
107 + num_blocks: int,
108 + dtype: torch.dtype,
109 +) -> None:
110 + num_slots = block_size * num_blocks
111 + slot_mapping = random.sample(range(num_slots), num_tokens)
112 + slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
113 +
114 + qkv = torch.randn(
115 + num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
116 + _, key, value = qkv.unbind(dim=1)
117 +
118 + qkv_clone = qkv.clone()
119 + _, cloned_key, cloned_value = qkv_clone.unbind(dim=1)
120 +
121 + x = 16 // torch.tensor([], dtype=dtype).element_size()
122 + key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
123 + key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
124 +
125 + value_cache_shape = (num_blocks, num_heads, head_size, block_size)
126 + value_cache = torch.randn(
127 + size=value_cache_shape, dtype=dtype, device='cuda')
128 +
129 + cache_ops.gather_cached_kv(key, value, key_cache, value_cache, slot_mapping)
130 +
131 + # Reference implementation.
132 + for i in range(num_tokens):
133 + reshaped_key = cloned_key.reshape(num_tokens, num_heads, head_size // x, x)
134 + block_idx = torch.div(slot_mapping[i], block_size, rounding_mode='floor')
135 + block_offset = slot_mapping[i] % block_size
136 + reshaped_key[i] = key_cache[block_idx, :, :, block_offset, :]
137 + cloned_value[i] = value_cache[block_idx, :, :, block_offset]
138 +
139 + assert torch.allclose(key, cloned_key)
140 + assert torch.allclose(value, cloned_value)
141 +
142 +
143 @torch.inference_mode()
144 def test_cache() -> None:
145 test_copy_blocks(
@@ -107,6 +148,9 @@ def test_cache() -> None:
148 test_reshape_and_cache(
149 num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
150 dtype=torch.half)
151 + test_gather_cached_kv(
152 + num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
153 + dtype=torch.half)
154
155
156 if __name__ == '__main__':