A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add cache engine
Woosuk Kwon committed
Feb 9, 2023 at 11:28 UTC
e7bee2aa811963b8c5ce352a427595749f6bfca1
1 file changed
+109
cacheflow/worker/cache_engine.py
new
+109
@@ -0,0 +1,109 @@
1
+from typing import List, Tuple
2
+
3
+import torch
4
+
5
+KVCache = Tuple[torch.Tensor, torch.Tensor]
6
+
7
+
8
+class CacheEngine:
9
+
10
+ def __init__(
11
+ self,
12
+ worker_id: int,
13
+ gpu_id: int,
14
+ num_layers: int,
15
+ num_heads: int,
16
+ head_size: int,
17
+ num_gpu_blocks: int,
18
+ num_cpu_blocks: int,
19
+ block_size: int,
20
+ dtype: torch.dtype = torch.float16,
21
+ ) -> None:
22
+ self.worker_id = worker_id
23
+ self.gpu_id = gpu_id
24
+ self.num_layers = num_layers
25
+ self.num_heads = num_heads
26
+ self.head_size = head_size
27
+ self.num_gpu_blocks = num_gpu_blocks
28
+ self.num_cpu_blocks = num_cpu_blocks
29
+ self.block_size = block_size
30
+ self.dtype = dtype
31
+
32
+ # Initialize the cache.
33
+ self.gpu_cache = self.allocate_gpu_cache()
34
+ self.cpu_cache = self.allocate_cpu_cache()
35
+
36
+ # Initialize the streams.
37
+ self.copy_stream = torch.cuda.Stream(device=gpu_id)
38
+ self.swap_stream = torch.cuda.Stream(device=gpu_id)
39
+ assert self.copy_stream != self.swap_stream
40
+ current_stream = torch.cuda.current_stream(device=gpu_id)
41
+ assert self.copy_stream != current_stream
42
+ assert self.swap_stream != current_stream
43
+
44
+ # Initialize the events for synchronization.
45
+
46
+ def allocate_gpu_cache(self) -> List[List[KVCache]]:
47
+ gpu_cache: List[List[KVCache]] = []
48
+ for _ in range(self.num_layers):
49
+ layer_cache: List[KVCache] = []
50
+ for _ in range(self.num_heads):
51
+ key_blocks = torch.empty(
52
+ (self.num_gpu_blocks, self.block_size * self.head_size),
53
+ dtype=self.dtype,
54
+ device=self.gpu_id,
55
+ )
56
+ value_blocks = torch.empty(
57
+ (self.num_gpu_blocks, self.block_size * self.head_size),
58
+ dtype=self.dtype,
59
+ device=self.gpu_id,
60
+ )
61
+ layer_cache.append((key_blocks, value_blocks))
62
+ gpu_cache.append(layer_cache)
63
+ return gpu_cache
64
+
65
+ def allocate_cpu_cache(self) -> List[List[KVCache]]:
66
+ cpu_cache: List[List[KVCache]] = []
67
+ for _ in range(self.num_layers):
68
+ layer_cache: List[KVCache] = []
69
+ for _ in range(self.num_heads):
70
+ key_blocks = torch.empty(
71
+ (self.num_cpu_blocks, self.block_size * self.head_size),
72
+ dtype=self.dtype,
73
+ pin_memory=True,
74
+ )
75
+ value_blocks = torch.empty(
76
+ (self.num_cpu_blocks, self.block_size * self.head_size),
77
+ dtype=self.dtype,
78
+ pin_memory=True,
79
+ )
80
+ layer_cache.append((key_blocks, value_blocks))
81
+ cpu_cache.append(layer_cache)
82
+ return cpu_cache
83
+
84
+ def copy(
85
+ self,
86
+ src_block_numbers: List[int],
87
+ dst_block_numbers: List[int],
88
+ ) -> None:
89
+ for layer in range(self.num_layers):
90
+ # TODO: Call the COPY op.
91
+ pass
92
+
93
+ def swap_out(
94
+ self,
95
+ gpu_block_numbers: List[int],
96
+ cpu_block_numbers: List[int],
97
+ ) -> None:
98
+ for layer in range(self.num_layers):
99
+ # TODO: Call the SWAP_OUT op on the swap stream.
100
+ pass
101
+
102
+ def swap_in(
103
+ self,
104
+ gpu_block_numbers: List[int],
105
+ cpu_block_numbers: List[int],
106
+ ) -> None:
107
+ for layer in range(self.num_layers):
108
+ # TODO: Call the SWAP_IN op on the swap stream.
109
+ pass