@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / af68ec1c5c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add tests for kernels

Woosuk Kwon committed Feb 18, 2023 at 19:23 UTC af68ec1c5c5eabb12865d8468391fe2c8df8d3b6
1 file changed +55
tests/kernels.py new
+55
@@ -0,0 +1,55 @@
1 +import random
2 +
3 +import torch
4 +
5 +from cacheflow.ops import reshape_and_cache
6 +
7 +
8 +def test_reshape_and_cache(
9 + num_tokens: int,
10 + num_heads: int,
11 + head_size: int,
12 + block_size: int,
13 + num_blocks: int,
14 + dtype: torch.dtype,
15 +) -> None:
16 + num_slots = block_size * num_blocks
17 + slot_mapping = random.sample(range(num_slots), num_tokens)
18 + slot_mapping = torch.tensor(slot_mapping, dtype=torch.int, device='cuda')
19 +
20 + kv_shape = (num_tokens, num_heads, head_size)
21 + key = torch.randn(size=kv_shape, dtype=dtype, device='cuda')
22 + value = torch.randn(size=kv_shape, dtype=dtype, device='cuda')
23 +
24 + x = 16 // torch.tensor([], dtype=dtype).element_size()
25 + key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
26 + key_cache = torch.randn(size=key_cache_shape, dtype=dtype, device='cuda')
27 + cloned_key_cache = key_cache.clone()
28 +
29 + value_cache_shape = (num_blocks, num_heads, block_size, head_size)
30 + value_cache = torch.randn(
31 + size=value_cache_shape, dtype=dtype, device='cuda')
32 + cloned_value_cache = value_cache.clone()
33 +
34 + reshape_and_cache(key, value, key_cache, value_cache, slot_mapping)
35 +
36 + for i in range(num_tokens):
37 + reshaped_key = key.reshape(num_tokens, num_heads, head_size // x, x)
38 + block_idx = slot_mapping[i] // block_size
39 + block_offset = slot_mapping[i] % block_size
40 + cloned_key_cache[block_idx, :, :, block_offset, :] = reshaped_key[i]
41 + cloned_value_cache[block_idx, :, block_offset, :] = value[i]
42 +
43 + assert torch.allclose(key_cache, cloned_key_cache)
44 + assert torch.allclose(value_cache, cloned_value_cache)
45 +
46 +
47 +@torch.no_grad()
48 +def test_kernels():
49 + test_reshape_and_cache(
50 + num_tokens=3, num_heads=2, head_size=16, block_size=2, num_blocks=2,
51 + dtype=torch.half)
52 +
53 +
54 +if __name__ == '__main__':
55 + test_kernels()