A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix can_swap_in
Woosuk Kwon committed
Feb 13, 2023 at 02:38 UTC
eb52db1bea23cdad8c5dc55d32e9eb69b20591d9
1 file changed
+10
-4
cacheflow/master/block_manager.py
+10
-4
@@ -70,13 +70,14 @@ class BlockSpaceManager:
70
self.block_tables: Dict[int, BlockTable] = {}
71
72
def can_allocate(self, seq_group: SequenceGroup) -> bool:
73
+ # NOTE: Here we assume that all sequences in the group have the same prompt.
74
seq = seq_group.seqs[0]
75
num_required_blocks = len(seq.logical_token_blocks)
76
num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
77
return num_required_blocks <= num_free_gpu_blocks
78
79
def allocate(self, seq_group: SequenceGroup) -> None:
79
- # Here, we assume that all sequences in the group have the same prompt.
80
+ # NOTE: Here we assume that all sequences in the group have the same prompt.
81
seq = seq_group.seqs[0]
82
83
# Allocate new physical token blocks that will store the prompt tokens.
@@ -124,10 +125,10 @@ class BlockSpaceManager:
125
self.gpu_allocator.free(last_block)
126
return last_block.block_number, new_block.block_number
127
127
- def fork(self, src_seq: Sequence, child_seq: Sequence) -> None:
128
+ def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
129
# NOTE: fork does not allocate a new physical block.
130
# Thus, it is always safe from OOM.
130
- src_block_table = self.block_tables[src_seq.seq_id]
131
+ src_block_table = self.block_tables[parent_seq.seq_id]
132
self.block_tables[child_seq.seq_id] = src_block_table.copy()
133
for block in src_block_table:
134
block.ref_count += 1
@@ -146,7 +147,12 @@ class BlockSpaceManager:
147
148
def can_swap_in(self, seq_group: SequenceGroup) -> bool:
149
blocks = self._get_physical_blocks(seq_group)
149
- return len(blocks) <= self.gpu_allocator.get_num_free_blocks()
150
+ num_running_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
151
+ num_free_blocks = self.gpu_allocator.get_num_free_blocks()
152
+ # NOTE: Conservatively, we assume that every sequence will allocate
153
+ # at least one free block right after the swap-in.
154
+ # NOTE: This should match the logic in can_append().
155
+ return len(blocks) + num_running_seqs <= num_free_blocks
156
157
def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
158
# src_block_number -> dst_block_number