A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Add block manager
Woosuk Kwon committed
Feb 9, 2023 at 11:27 UTC
aa78aeaa0f4fee0ccb0184914266d6170cfc848f
1 file changed
+204
cacheflow/master/block_manager.py
new
+204
@@ -0,0 +1,204 @@
1
+from typing import Dict, Iterable, List, Optional, Set, Tuple
2
+
3
+from cacheflow.block import PhysicalTokenBlock
4
+from cacheflow.sequence import Sequence
5
+from cacheflow.sequence import SequenceGroup
6
+from cacheflow.sequence import SequenceStatus
7
+from cacheflow.utils import Device
8
+
9
+
10
+class BlockAllocator:
11
+
12
+ def __init__(
13
+ self,
14
+ device: Device,
15
+ block_size: int,
16
+ num_blocks: int,
17
+ ) -> None:
18
+ assert block_size in [8, 16, 32]
19
+ self.device = device
20
+ self.block_size = block_size
21
+ self.num_blocks = num_blocks
22
+
23
+ # Initialize the free blocks.
24
+ # TODO(woosuk): Make this a priority queue.
25
+ self.free_blocks = [
26
+ PhysicalTokenBlock(device=device, block_number=i, block_size=block_size)
27
+ for i in range(num_blocks)
28
+ ]
29
+
30
+ def allocate(self) -> PhysicalTokenBlock:
31
+ if not self.free_blocks:
32
+ raise ValueError('Out of memory! '
33
+ f'No more free blocks are available.')
34
+ block = self.free_blocks.pop()
35
+ block.ref_count = 1
36
+ return block
37
+
38
+ def free(self, block: PhysicalTokenBlock) -> None:
39
+ if block.ref_count == 0:
40
+ raise ValueError('Double free! '
41
+ f'The block {block} is already freed.')
42
+ block.ref_count -= 1
43
+ if block.ref_count == 0:
44
+ self.free_blocks.append(block)
45
+
46
+ def get_num_free_blocks(self) -> int:
47
+ return len(self.free_blocks)
48
+
49
+
50
+# Mapping: logical block number -> physical block.
51
+BlockTable = List[PhysicalTokenBlock]
52
+
53
+
54
+class BlockSpaceManager:
55
+
56
+ def __init__(
57
+ self,
58
+ block_size: int,
59
+ num_gpu_blocks: int,
60
+ num_cpu_blocks: int,
61
+ ) -> None:
62
+ self.block_size = block_size
63
+ self.num_total_gpu_blocks = num_gpu_blocks
64
+ self.num_total_cpu_blocks = num_cpu_blocks
65
+
66
+ self.gpu_allocator = BlockAllocator(Device.GPU, block_size, num_gpu_blocks)
67
+ self.cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
68
+
69
+ # Mapping: seq_id -> BlockTable.
70
+ self.block_tables: Dict[int, BlockTable] = {}
71
+
72
+ def can_allocate(self, seq_group: SequenceGroup) -> bool:
73
+ seq = seq_group.seqs[0]
74
+ num_required_blocks = len(seq.logical_token_blocks)
75
+ num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
76
+ return num_required_blocks <= num_free_gpu_blocks
77
+
78
+ def allocate(self, seq_group: SequenceGroup) -> None:
79
+ # Here, we assume that all sequences in the group have the same prompt.
80
+ seq = seq_group.seqs[0]
81
+
82
+ # Allocate new physical token blocks that will store the prompt tokens.
83
+ block_table: BlockTable = []
84
+ for _ in range(len(seq.logical_token_blocks)):
85
+ block = self.gpu_allocator.allocate()
86
+ # Set the reference counts of the token blocks.
87
+ block.ref_count = seq_group.num_seqs()
88
+ block_table.append(block)
89
+
90
+ # Assign the block table for each sequence.
91
+ for seq in seq_group.seqs:
92
+ self.block_tables[seq.seq_id] = block_table.copy()
93
+
94
+ def can_append(self, seq_group: SequenceGroup) -> bool:
95
+ # Simple heuristic: If there is at least one free block
96
+ # for each sequence, we can append.
97
+ num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
98
+ num_seqs = seq_group.num_seqs(status=SequenceStatus.SERVING)
99
+ return num_seqs <= num_free_gpu_blocks
100
+
101
+ def append(self, seq: Sequence) -> Optional[Tuple[int, int]]:
102
+ """Allocate a physical slot for the new token."""
103
+ logical_blocks = seq.logical_token_blocks
104
+ block_table = self.block_tables[seq.seq_id]
105
+
106
+ if len(block_table) < len(logical_blocks):
107
+ # The sequence has a new logical block.
108
+ # Allocate a new physical block.
109
+ block = self.gpu_allocator.allocate()
110
+ block_table.append(block)
111
+ return None
112
+
113
+ # We want to append the token to the last physical block.
114
+ last_block = block_table[-1]
115
+ assert last_block.device == Device.GPU
116
+ if last_block.ref_count == 1:
117
+ # Append.
118
+ return None
119
+ else:
120
+ # The last block is shared with other sequences.
121
+ # Copy on Write: Allocate a new block and copy the tokens.
122
+ block = self.gpu_allocator.allocate()
123
+ block_table.append(block)
124
+ self.gpu_allocator.free(last_block)
125
+ return last_block.block_number, block.block_number
126
+
127
+ def fork(self, src_seq: Sequence, child_seq: Sequence) -> None:
128
+ # NOTE: fork does not allocate a new physical block.
129
+ # Thus, it is always safe from OOM.
130
+ src_block_table = self.block_tables[src_seq.seq_id]
131
+ self.block_tables[child_seq.seq_id] = src_block_table.copy()
132
+ for block in src_block_table:
133
+ block.ref_count += 1
134
+
135
+ def _get_physical_blocks(self, seq_group: SequenceGroup) -> List[PhysicalTokenBlock]:
136
+ # NOTE: Here, we assume that the physical blocks are only shared by
137
+ # the sequences in the same group.
138
+ blocks: Set[PhysicalTokenBlock] = set()
139
+ for seq in seq_group.seqs:
140
+ if seq.status == SequenceStatus.FINISHED:
141
+ continue
142
+ block_table = self.block_tables[seq.seq_id]
143
+ for block in block_table:
144
+ blocks.add(block)
145
+ return list(blocks)
146
+
147
+ def can_swap_in(self, seq_group: SequenceGroup) -> bool:
148
+ blocks = self._get_physical_blocks(seq_group)
149
+ return len(blocks) <= self.gpu_allocator.get_num_free_blocks()
150
+
151
+ def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
152
+ # src_block_number -> dst_block_number
153
+ mapping: Dict[int, int] = {}
154
+ for seq in seq_group.seqs:
155
+ if seq.status == SequenceStatus.FINISHED:
156
+ continue
157
+ block_table = self.block_tables[seq.seq_id]
158
+
159
+ for cpu_block in block_table:
160
+ if cpu_block in mapping:
161
+ continue
162
+ gpu_block = self.gpu_allocator.allocate()
163
+ mapping[cpu_block.block_number] = gpu_block.block_number
164
+ # Free the CPU block swapped in to GPU.
165
+ self.cpu_allocator.free(cpu_block)
166
+ return mapping
167
+
168
+ def can_swap_out(self, seq_group: SequenceGroup) -> bool:
169
+ blocks = self._get_physical_blocks(seq_group)
170
+ return len(blocks) <= self.cpu_allocator.get_num_free_blocks()
171
+
172
+ def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
173
+ # src_block_number -> dst_block_number
174
+ mapping: Dict[int, int] = {}
175
+ for seq in seq_group.seqs:
176
+ if seq.status == SequenceStatus.FINISHED:
177
+ continue
178
+ block_table = self.block_tables[seq.seq_id]
179
+
180
+ for gpu_block in block_table:
181
+ if gpu_block.block_number in mapping:
182
+ continue
183
+ cpu_block = self.cpu_allocator.allocate()
184
+ mapping[gpu_block.block_number] = cpu_block.block_number
185
+ # Free the GPU block swapped out to CPU.
186
+ self.gpu_allocator.free(gpu_block)
187
+ return mapping
188
+
189
+ def _free_blocks(self, blocks: Iterable[PhysicalTokenBlock]) -> None:
190
+ for block in blocks:
191
+ if block.device == Device.GPU:
192
+ self.gpu_allocator.free(block)
193
+ else:
194
+ self.cpu_allocator.free(block)
195
+
196
+ def free(self, seq: Sequence) -> None:
197
+ block_table = self.block_tables[seq.seq_id]
198
+ self._free_blocks(block_table)
199
+ del self.block_tables[seq.seq_id]
200
+
201
+ def reset(self) -> None:
202
+ for block_table in self.block_tables.values():
203
+ self._free_blocks(block_table)
204
+ self.block_tables.clear()