A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Memcpy kernel for flash attention (#29)
* optimize * add benchmark * add assert * add test
Siyuan (Ryans) Zhuang committed
Apr 10, 2023 at 18:22 UTC
e3cec88aa5b7ac391e4aa6dc9b6388100d59d8f9
4 files changed
+293
benchmark/benchmark_cache.py
new
+81
@@ -0,0 +1,81 @@
1
+import functools
2
+import random
3
+import time
4
+
5
+import torch
6
+
7
+from cacheflow import cache_ops
8
+
9
+
10
+def benchmark(name, f, size: int, num_warmup = 10, num_iters = 100):
11
+ for _ in range(num_warmup):
12
+ f()
13
+ torch.cuda.synchronize()
14
+
15
+ start = time.time()
16
+ for _ in range(num_iters):
17
+ f()
18
+ torch.cuda.synchronize()
19
+ end = time.time()
20
+ avg_time = (end - start) / num_iters
21
+ print(f'[Latency] {name}: {avg_time * 1000:.3f} ms')
22
+ print(f'[Throughput] {name}: {size / avg_time / 2 ** 30:.3f} GB/s')
23
+
24
+
25
+@torch.inference_mode()
26
+def test_gather_cached_kv(
27
+ num_tokens: int,
28
+ num_heads: int,
29
+ head_size: int,
30
+ block_size: int,
31
+ num_blocks: int,
32
+ dtype: torch.dtype,
33
+) -> None:
34
+ print(f'num_tokens: {num_tokens}, num_heads: {num_heads}, '
35
+ f'head_size: {head_size}, block_size: {block_size}, '
36
+ f'num_blocks: {num_blocks}, dtype: {dtype}')
37
+
38
+ num_slots = block_size * num_blocks
39
+ slot_mapping = random.sample(range(num_slots), num_tokens)
40
+ slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
41
+
42
+ qkv = torch.randn(
43
+ num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
44
+ _, key, value = qkv.unbind(dim=1)
45
+
46
+ x = 16 // torch.tensor([], dtype=dtype).element_size()
47
+ key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
48
+ key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
49
+
50
+ value_cache_shape = (num_blocks, num_heads, head_size, block_size)
51
+ value_cache = torch.randn(
52
+ size=value_cache_shape, dtype=dtype, device='cuda')
53
+
54
+ # Run Flash attention.
55
+ def run():
56
+ cache_ops.gather_cached_kv(key, value, key_cache, value_cache, slot_mapping)
57
+
58
+ benchmark('gather_cached_kv', run,
59
+ size=num_tokens * num_heads * head_size * 2 * qkv.element_size())
60
+
61
+
62
+if __name__ == '__main__':
63
+ BLOCK_SIZE = 8
64
+ NUM_BLOCKS = 1024
65
+ DTYPE = torch.half
66
+
67
+ # LLaMA-13B and OPT-13B
68
+ NUM_HEADS = 40
69
+ HEAD_SIZE = 128
70
+
71
+ run_benchmark = functools.partial(
72
+ test_gather_cached_kv,
73
+ num_heads=NUM_HEADS,
74
+ head_size=HEAD_SIZE,
75
+ block_size=BLOCK_SIZE,
76
+ num_blocks=NUM_BLOCKS,
77
+ dtype=DTYPE,
78
+ )
79
+
80
+ for i in range(6, 12):
81
+ run_benchmark(num_tokens=2 ** i)
csrc/cache.cpp
+11
@@ -20,6 +20,13 @@ void reshape_and_cache(
20
torch::Tensor& value_cache,
21
torch::Tensor& slot_mapping);
22
23
+void gather_cached_kv(
24
+ torch::Tensor& key,
25
+ torch::Tensor& value,
26
+ torch::Tensor& key_cache,
27
+ torch::Tensor& value_cache,
28
+ torch::Tensor& slot_mapping);
29
+
30
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
31
m.def(
32
"swap_blocks",
@@ -33,4 +40,8 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
40
"reshape_and_cache",
41
&reshape_and_cache,
42
"Reshape the key and value tensors and cache them");
43
+ m.def(
44
+ "gather_cached_kv",
45
+ &gather_cached_kv,
46
+ "Gather key and value from the cache into contiguous QKV tensors");
47
}
csrc/cache_kernels.cu
+157
@@ -176,6 +176,124 @@ __global__ void reshape_and_cache_kernel(
176
}
177
}
178
179
+// Grid: (num_blocks, block_size).
180
+template<typename scalar_t>
181
+__global__ void gather_cached_kv_kernel(
182
+ scalar_t* __restrict__ key, // [num_tokens, [stride], num_heads, head_size]
183
+ scalar_t* __restrict__ value, // [num_tokens, [stride], num_heads, head_size]
184
+ const scalar_t* __restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
185
+ const scalar_t* __restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
186
+ const int* __restrict__ slot_mapping, // [num_tokens]
187
+ const int key_stride,
188
+ const int value_stride,
189
+ const int num_heads,
190
+ const int head_size,
191
+ const int block_size,
192
+ const int x) {
193
+ const int token_idx = blockIdx.x;
194
+ const int slot_idx = slot_mapping[token_idx];
195
+ const int block_idx = slot_idx / block_size;
196
+ const int block_offset = slot_idx % block_size;
197
+
198
+ const int num_tokens = num_heads * head_size;
199
+ for (int i = threadIdx.x; i < num_tokens; i += blockDim.x) {
200
+ const int tgt_key_idx = token_idx * key_stride + i;
201
+ const int tgt_value_idx = token_idx * value_stride + i;
202
+
203
+ const int head_idx = i / head_size;
204
+ const int head_offset = i % head_size;
205
+ const int x_idx = head_offset / x; // the offset of the [head_size/x] dimension
206
+ const int x_offset = head_offset % x;
207
+
208
+ const int src_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
209
+ + head_idx * (head_size / x) * block_size * x
210
+ + x_idx * block_size * x
211
+ + block_offset * x
212
+ + x_offset;
213
+ const int src_value_idx = block_idx * num_heads * head_size * block_size
214
+ + head_idx * head_size * block_size
215
+ + head_offset * block_size
216
+ + block_offset;
217
+
218
+ key[tgt_key_idx] = __ldg(&key_cache[src_key_idx]);
219
+ value[tgt_value_idx] = __ldg(&value_cache[src_value_idx]);
220
+ }
221
+}
222
+
223
+template <typename scalar_t>
224
+__global__ void gather_cached_kv_kernel_optimized(
225
+ scalar_t *__restrict__ key, // [num_tokens, [stride], num_heads, head_size]
226
+ scalar_t *__restrict__ value, // [num_tokens, [stride], num_heads, head_size]
227
+ const scalar_t *__restrict__ key_cache, // [num_blocks, num_heads, head_size/x, block_size, x]
228
+ const scalar_t *__restrict__ value_cache, // [num_blocks, num_heads, head_size, block_size]
229
+ const int *__restrict__ slot_mapping, // [num_tokens]
230
+ const int key_stride,
231
+ const int value_stride,
232
+ const int num_heads,
233
+ const int head_size,
234
+ const int block_size,
235
+ const int x)
236
+{
237
+ const int token_idx = blockIdx.x;
238
+ const int slot_idx = slot_mapping[token_idx];
239
+ const int block_idx = slot_idx / block_size;
240
+ const int block_offset = slot_idx % block_size;
241
+
242
+ const int dim = num_heads * head_size;
243
+ assert(dim % 4 == 0); // this is true for known use cases
244
+ const int unroll_factor = 4;
245
+ const int unrolled_dim = dim / unroll_factor;
246
+
247
+ for (int i = threadIdx.x; i < unrolled_dim; i += blockDim.x)
248
+ {
249
+ int tgt_key_indices[unroll_factor];
250
+ int tgt_value_indices[unroll_factor];
251
+ int src_key_indices[unroll_factor];
252
+ int src_value_indices[unroll_factor];
253
+ scalar_t keys_to_store[unroll_factor];
254
+ scalar_t values_to_store[unroll_factor];
255
+
256
+ #pragma unroll
257
+ for (int j = 0; j < unroll_factor; ++j)
258
+ {
259
+ int index = i + j * unrolled_dim;
260
+
261
+ const int tgt_key_idx = token_idx * key_stride + index;
262
+ const int tgt_value_idx = token_idx * value_stride + index;
263
+
264
+ const int head_idx = index / head_size;
265
+ const int head_offset = index % head_size;
266
+ const int x_idx = head_offset / x;
267
+ const int x_offset = head_offset % x;
268
+
269
+ const int src_key_idx = block_idx * num_heads * (head_size / x) * block_size * x
270
+ + head_idx * (head_size / x) * block_size * x
271
+ + x_idx * block_size * x
272
+ + block_offset * x
273
+ + x_offset;
274
+ const int src_value_idx = block_idx * num_heads * head_size * block_size
275
+ + head_idx * head_size * block_size
276
+ + head_offset * block_size
277
+ + block_offset;
278
+
279
+ tgt_key_indices[j] = tgt_key_idx;
280
+ tgt_value_indices[j] = tgt_value_idx;
281
+ src_key_indices[j] = src_key_idx;
282
+ src_value_indices[j] = src_value_idx;
283
+
284
+ keys_to_store[j] = __ldg(&key_cache[src_key_idx]);
285
+ values_to_store[j] = __ldg(&value_cache[src_value_idx]);
286
+ }
287
+
288
+ #pragma unroll
289
+ for (int j = 0; j < unroll_factor; ++j)
290
+ {
291
+ key[tgt_key_indices[j]] = keys_to_store[j];
292
+ value[tgt_value_indices[j]] = values_to_store[j];
293
+ }
294
+ }
295
+}
296
+
297
} // namespace cacheflow
298
299
void reshape_and_cache(
@@ -215,3 +333,42 @@ void reshape_and_cache(
333
x);
334
});
335
}
336
+
337
+
338
+void gather_cached_kv(
339
+ torch::Tensor& key, // [out] [num_tokens, num_heads, head_size]
340
+ torch::Tensor& value, // [out] [num_tokens, num_heads, head_size]
341
+ torch::Tensor& key_cache, // [in] [num_blocks, num_heads, head_size/x, block_size, x]
342
+ torch::Tensor& value_cache, // [in] [num_blocks, num_heads, head_size, block_size]
343
+ torch::Tensor& slot_mapping) // [in] [num_tokens]
344
+{
345
+ int num_tokens = key.size(0);
346
+ int num_heads = key.size(1);
347
+ int head_size = key.size(2);
348
+ int block_size = key_cache.size(3);
349
+ int x = key_cache.size(4);
350
+
351
+ int key_stride = key.stride(0);
352
+ int value_stride = value.stride(0);
353
+
354
+ dim3 grid(num_tokens);
355
+ dim3 block(std::min(num_heads * head_size, 512));
356
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
357
+ AT_DISPATCH_FLOATING_TYPES_AND_HALF(
358
+ key.scalar_type(),
359
+ "gather_cached_kv_kernel_optimized",
360
+ [&] {
361
+ cacheflow::gather_cached_kv_kernel_optimized<scalar_t><<<grid, block, 0, stream>>>(
362
+ key.data_ptr<scalar_t>(),
363
+ value.data_ptr<scalar_t>(),
364
+ key_cache.data_ptr<scalar_t>(),
365
+ value_cache.data_ptr<scalar_t>(),
366
+ slot_mapping.data_ptr<int>(),
367
+ key_stride,
368
+ value_stride,
369
+ num_heads,
370
+ head_size,
371
+ block_size,
372
+ x);
373
+ });
374
+}
tests/kernels/cache.py
+44
@@ -99,6 +99,47 @@ def test_reshape_and_cache(
99
assert torch.allclose(value_cache, cloned_value_cache)
100
101
102
+def test_gather_cached_kv(
103
+ num_tokens: int,
104
+ num_heads: int,
105
+ head_size: int,
106
+ block_size: int,
107
+ num_blocks: int,
108
+ dtype: torch.dtype,
109
+) -> None:
110
+ num_slots = block_size * num_blocks
111
+ slot_mapping = random.sample(range(num_slots), num_tokens)
112
+ slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
113
+
114
+ qkv = torch.randn(
115
+ num_tokens, 3, num_heads, head_size, dtype=dtype, device='cuda')
116
+ _, key, value = qkv.unbind(dim=1)
117
+
118
+ qkv_clone = qkv.clone()
119
+ _, cloned_key, cloned_value = qkv_clone.unbind(dim=1)
120
+
121
+ x = 16 // torch.tensor([], dtype=dtype).element_size()
122
+ key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
123
+ key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
124
+
125
+ value_cache_shape = (num_blocks, num_heads, head_size, block_size)
126
+ value_cache = torch.randn(
127
+ size=value_cache_shape, dtype=dtype, device='cuda')
128
+
129
+ cache_ops.gather_cached_kv(key, value, key_cache, value_cache, slot_mapping)
130
+
131
+ # Reference implementation.
132
+ for i in range(num_tokens):
133
+ reshaped_key = cloned_key.reshape(num_tokens, num_heads, head_size // x, x)
134
+ block_idx = torch.div(slot_mapping[i], block_size, rounding_mode='floor')
135
+ block_offset = slot_mapping[i] % block_size
136
+ reshaped_key[i] = key_cache[block_idx, :, :, block_offset, :]
137
+ cloned_value[i] = value_cache[block_idx, :, :, block_offset]
138
+
139
+ assert torch.allclose(key, cloned_key)
140
+ assert torch.allclose(value, cloned_value)
141
+
142
+
143
@torch.inference_mode()
144
def test_cache() -> None:
145
test_copy_blocks(
@@ -107,6 +148,9 @@ def test_cache() -> None:
148
test_reshape_and_cache(
149
num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
150
dtype=torch.half)
151
+ test_gather_cached_kv(
152
+ num_tokens=3, num_heads=2, head_size=16, block_size=8, num_blocks=2,
153
+ dtype=torch.half)
154
155
156
if __name__ == '__main__':