A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add tests for kernels
Woosuk Kwon committed
Feb 18, 2023 at 19:23 UTC
af68ec1c5c5eabb12865d8468391fe2c8df8d3b6
1 file changed
+55
tests/kernels.py
new
+55
@@ -0,0 +1,55 @@
1
+import random
2
+
3
+import torch
4
+
5
+from cacheflow.ops import reshape_and_cache
6
+
7
+
8
+def test_reshape_and_cache(
9
+ num_tokens: int,
10
+ num_heads: int,
11
+ head_size: int,
12
+ block_size: int,
13
+ num_blocks: int,
14
+ dtype: torch.dtype,
15
+) -> None:
16
+ num_slots = block_size * num_blocks
17
+ slot_mapping = random.sample(range(num_slots), num_tokens)
18
+ slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
19
+
20
+ kv_shape = (num_tokens, num_heads, head_size)
21
+ key = torch.randn(size=kv_shape, dtype=dtype, device='cuda')
22
+ value = torch.randn(size=kv_shape, dtype=dtype, device='cuda')
23
+
24
+ x = 16 // torch.tensor([], dtype=dtype).element_size()
25
+ key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
26
+ key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
27
+ cloned_key_cache = key_cache.clone()
28
+
29
+ value_cache_shape = (num_blocks, num_heads, block_size, head_size)
30
+ value_cache = torch.randn(
31
+ size=value_cache_shape, dtype=dtype, device='cuda')
32
+ cloned_value_cache = value_cache.clone()
33
+
34
+ reshape_and_cache(key, value, key_cache, value_cache, slot_mapping)
35
+
36
+ for i in range(num_tokens):
37
+ reshaped_key = key.reshape(num_tokens, num_heads, head_size // x, x)
38
+ block_idx = slot_mapping[i] // block_size
39
+ block_offset = slot_mapping[i] % block_size
40
+ cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
41
+ cloned_value_cache[block_idx, :, block_offset, :] = value[i]
42
+
43
+ assert torch.allclose(key_cache, cloned_key_cache)
44
+ assert torch.allclose(value_cache, cloned_value_cache)
45
+
46
+
47
+@torch.no_grad()
48
+def test_kernels():
49
+ test_reshape_and_cache(
50
+ num_tokens=3, num_heads=2, head_size=16, block_size=2, num_blocks=2,
51
+ dtype=torch.half)
52
+
53
+
54
+if __name__ == '__main__':
55
+ test_kernels()