@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eb52db1bea
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix can_swap_in

Woosuk Kwon committed Feb 13, 2023 at 02:38 UTC eb52db1bea23cdad8c5dc55d32e9eb69b20591d9
1 file changed +10 -4
cacheflow/master/block_manager.py
+10 -4
@@ -70,13 +70,14 @@ class BlockSpaceManager:
70 self.block_tables: Dict[int, BlockTable] = {}
71
72 def can_allocate(self, seq_group: SequenceGroup) -> bool:
73 + # NOTE: Here we assume that all sequences in the group have the same prompt.
74 seq = seq_group.seqs[0]
75 num_required_blocks = len(seq.logical_token_blocks)
76 num_free_gpu_blocks = self.gpu_allocator.get_num_free_blocks()
77 return num_required_blocks <= num_free_gpu_blocks
78
79 def allocate(self, seq_group: SequenceGroup) -> None:
79 - # Here, we assume that all sequences in the group have the same prompt.
80 + # NOTE: Here we assume that all sequences in the group have the same prompt.
81 seq = seq_group.seqs[0]
82
83 # Allocate new physical token blocks that will store the prompt tokens.
@@ -124,10 +125,10 @@ class BlockSpaceManager:
125 self.gpu_allocator.free(last_block)
126 return last_block.block_number, new_block.block_number
127
127 - def fork(self, src_seq: Sequence, child_seq: Sequence) -> None:
128 + def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
129 # NOTE: fork does not allocate a new physical block.
130 # Thus, it is always safe from OOM.
130 - src_block_table = self.block_tables[src_seq.seq_id]
131 + src_block_table = self.block_tables[parent_seq.seq_id]
132 self.block_tables[child_seq.seq_id] = src_block_table.copy()
133 for block in src_block_table:
134 block.ref_count += 1
@@ -146,7 +147,12 @@ class BlockSpaceManager:
147
148 def can_swap_in(self, seq_group: SequenceGroup) -> bool:
149 blocks = self._get_physical_blocks(seq_group)
149 - return len(blocks) <= self.gpu_allocator.get_num_free_blocks()
150 + num_running_seqs = seq_group.num_seqs(status=SequenceStatus.RUNNING)
151 + num_free_blocks = self.gpu_allocator.get_num_free_blocks()
152 + # NOTE: Conservatively, we assume that every sequence will allocate
153 + # at least one free block right after the swap-in.
154 + # NOTE: This should match the logic in can_append().
155 + return len(blocks) + num_running_seqs <= num_free_blocks
156
157 def swap_in(self, seq_group: SequenceGroup) -> Dict[int, int]:
158 # src_block_number -> dst_block_number