@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / aa78aeaa0f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Add block manager

Woosuk Kwon committed Feb 9, 2023 at 11:27 UTC aa78aeaa0f4fee0ccb0184914266d6170cfc848f
1 file changed +204
cacheflow/master/block_manager.py new
+204
@@ -0,0 +1,204 @@
1 +from typing import Dict, Iterable, List, Optional, Set, Tuple
2 +
3 +from cacheflow.block import PhysicalTokenBlock
4 +from cacheflow.sequence import Sequence
5 +from cacheflow.sequence import SequenceGroup
6 +from cacheflow.sequence import SequenceStatus
7 +from cacheflow.utils import Device
8 +
9 +
10 +class BlockAllocator:
11 +
12 + def __init__(
13 + self,
14 + device: Device,
15 + block_size: int,
16 + num_blocks: int,
17 + ) -> None:
18 + assert block_size in [8, 16, 32]
19 + self.device = device
20 + self.block_size = block_size
21 + self.num_blocks = num_blocks
22 +
23 + # Initialize the free blocks.
24 + # TODO(woosuk): Make this a priority queue.
25 + self.free_blocks = [
26 + PhysicalTokenBlock(device=device, block_number=i, block_size=block_size)
27 + for i in range(num_blocks)
28 + ]
29 +
30 + def allocate(self) -> PhysicalTokenBlock:
31 + if not self.free_blocks:
32 + raise ValueError('Out of memory! '
33 + f'No more free blocks are available.')
34 + block = self.free_blocks.pop()
35 + block.ref_count = 1
36 + return block
37 +
38 + def free(self, block: PhysicalTokenBlock) -> None:
39 + if block.ref_count == 0:
40 + raise ValueError('Double free! '
41 + f'The block {block} is already freed.')
42 + block.ref_count -= 1
43 + if block.ref_count == 0:
44 + self.free_blocks.append(block)
45 +
46 + def get_num_free_blocks(self) -> int:
47 + return len(self.free_blocks)
48 +
49 +
50 +# Mapping: logical block number -> physical block.
51 +BlockTable = List[PhysicalTokenBlock]
52 +
53 +
54 +class BlockSpaceManager:
55 +
56 + def __init__(
57 + self,
58 + block_size: int,
59 + num_gpu_blocks: int,
60 + num_cpu_blocks: int,
61 + ) -> None:
62 + self.block_size = block_size
63 + self.num_total_gpu_blocks = num_gpu_blocks
64 + self.num_total_cpu_blocks = num_cpu_blocks
65 +
66 + self.gpu_allocator = BlockAllocator(Device.GPU, block_size, num_gpu_blocks)
67 + self.cpu_allocator = BlockAllocator(Device.CPU, block_size, num_cpu_blocks)
68 +
69 + # Mapping: seq_id -> BlockTable.
70 + self.block_tables: Dict[int, BlockTable] = {}
71 +
72 + def can_allocate(self, seq_group: SequenceGroup) -> bool:
73 + seq = seq_group.seqs[0]
74 + num_required_blocks = len(seq.logical_token_blocks)
75 + num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
76 + return num_required_blocks <= num_free_gpu_blocks
77 +
78 + def allocate(self, seq_group: SequenceGroup) -> None:
79 + # Here, we assume that all sequences in the group have the same prompt.
80 + seq = seq_group.seqs[0]
81 +
82 + # Allocate new physical token blocks that will store the prompt tokens.
83 + block_table: BlockTable = []
84 + for _ in range(len(seq.logical_token_blocks)):
85 + block = self.gpu_allocator.allocate()
86 + # Set the reference counts of the token blocks.
87 + block.ref_count = seq_group.num_seqs()
88 + block_table.append(block)
89 +
90 + # Assign the block table for each sequence.
91 + for seq in seq_group.seqs:
92 + self.block_tables[seq.seq_id] = block_table.copy()
93 +
94 + def can_append(self, seq_group: SequenceGroup) -> bool:
95 + # Simple heuristic: If there is at least one free block
96 + # for each sequence, we can append.
97 + num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
98 + num_seqs = seq_group.num_seqs(status=SequenceStatus.SERVING)
99 + return num_seqs <= num_free_gpu_blocks
100 +
101 + def append(self, seq: Sequence) -> Optional[Tuple[int, int]]:
102 + """Allocate a physical slot for the new token."""
103 + logical_blocks = seq.logical_token_blocks
104 + block_table = self.block_tables[seq.seq_id]
105 +
106 + if len(block_table) < len(logical_blocks):
107 + # The sequence has a new logical block.
108 + # Allocate a new physical block.
109 + block = self.gpu_allocator.allocate()
110 + block_table.append(block)
111 + return None
112 +
113 + # We want to append the token to the last physical block.
114 + last_block = block_table[-1]
115 + assert last_block.device == Device.GPU
116 + if last_block.ref_count == 1:
117 + # Append.
118 + return None
119 + else:
120 + # The last block is shared with other sequences.
121 + # Copy on Write: Allocate a new block and copy the tokens.
122 + block = self.gpu_allocator.allocate()
123 + block_table.append(block)
124 + self.gpu_allocator.free(last_block)
125 + return last_block.block_number, block.block_number
126 +
127 + def fork(self, src_seq: Sequence, child_seq: Sequence) -> None:
128 + # NOTE: fork does not allocate a new physical block.
129 + # Thus, it is always safe from OOM.
130 + src_block_table = self.block_tables[src_seq.seq_id]
131 + self.block_tables[child_seq.seq_id] = src_block_table.copy()
132 + for block in src_block_table:
133 + block.ref_count += 1
134 +
135 + def _get_physical_blocks(self, seq_group: SequenceGroup) -> List[PhysicalTokenBlock]:
136 + # NOTE: Here, we assume that the physical blocks are only shared by
137 + # the sequences in the same group.
138 + blocks: Set[PhysicalTokenBlock] = set()
139 + for seq in seq_group.seqs:
140 + if seq.status == SequenceStatus.FINISHED:
141 + continue
142 + block_table = self.block_tables[seq.seq_id]
143 + for block in block_table:
144 + blocks.add(block)
145 + return list(blocks)
146 +
147 + def can_swap_in(self, seq_group: SequenceGroup) -> bool:
148 + blocks = self._get_physical_blocks(seq_group)
149 + return len(blocks) <= self.gpu_allocator.get_num_free_blocks()
150 +
151 + def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
152 + # src_block_number -> dst_block_number
153 + mapping: Dict[int, int] = {}
154 + for seq in seq_group.seqs:
155 + if seq.status == SequenceStatus.FINISHED:
156 + continue
157 + block_table = self.block_tables[seq.seq_id]
158 +
159 + for cpu_block in block_table:
160 + if cpu_block in mapping:
161 + continue
162 + gpu_block = self.gpu_allocator.allocate()
163 + mapping[cpu_block.block_number] = gpu_block.block_number
164 + # Free the CPU block swapped in to GPU.
165 + self.cpu_allocator.free(cpu_block)
166 + return mapping
167 +
168 + def can_swap_out(self, seq_group: SequenceGroup) -> bool:
169 + blocks = self._get_physical_blocks(seq_group)
170 + return len(blocks) <= self.cpu_allocator.get_num_free_blocks()
171 +
172 + def swap_out(self, seq_group: SequenceGroup) -> Dict[int, int]:
173 + # src_block_number -> dst_block_number
174 + mapping: Dict[int, int] = {}
175 + for seq in seq_group.seqs:
176 + if seq.status == SequenceStatus.FINISHED:
177 + continue
178 + block_table = self.block_tables[seq.seq_id]
179 +
180 + for gpu_block in block_table:
181 + if gpu_block.block_number in mapping:
182 + continue
183 + cpu_block = self.cpu_allocator.allocate()
184 + mapping[gpu_block.block_number] = cpu_block.block_number
185 + # Free the GPU block swapped out to CPU.
186 + self.gpu_allocator.free(gpu_block)
187 + return mapping
188 +
189 + def _free_blocks(self, blocks: Iterable[PhysicalTokenBlock]) -> None:
190 + for block in blocks:
191 + if block.device == Device.GPU:
192 + self.gpu_allocator.free(block)
193 + else:
194 + self.cpu_allocator.free(block)
195 +
196 + def free(self, seq: Sequence) -> None:
197 + block_table = self.block_tables[seq.seq_id]
198 + self._free_blocks(block_table)
199 + del self.block_tables[seq.seq_id]
200 +
201 + def reset(self) -> None:
202 + for block_table in self.block_tables.values():
203 + self._free_blocks(block_table)
204 + self.block_tables.clear()